diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 40e08146f8..aee807b20d 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -13,7 +13,8 @@ RUN apt-get -y update \ libnl-route-3-dev=3.4.* \ make=4.2.* \ pkg-config=0.29-6 \ - protobuf-compiler=3.6.* + protobuf-compiler=3.6.* \ + golang-go RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 diff --git a/Dockerfile b/Dockerfile index 4b5926f4a2..5655c04af7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -76,7 +76,7 @@ RUN apt-get update \ make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \ libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \ libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \ - libv8-dev tesseract-ocr \ + libv8-dev tesseract-ocr golang-go \ && rm -rf /var/lib/apt/lists/* ENV TZ=Etc/UTC diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 2b9aeb7a70..bc5b523f39 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -4134,6 +4134,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +[[package]] +name = "unicode-general-category" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1218098468b8085b19a2824104c70d976491d247ce194bbd9dc77181150cdfd6" + [[package]] name = "unicode-id" version = "0.3.2" @@ -4520,6 +4526,7 @@ dependencies = [ "tracing", "tracing-subscriber", "ulid", + "unicode-general-category", "url", "urlencoding", "uuid", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 6bda5e9d25..2f653d5172 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -58,6 +58,7 @@ async-recursion = "^1" swc_common = "^0" swc_ecma_parser = "^0" swc_ecma_ast = "^0" +unicode-general-category = "^0" sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]} dotenv = "^0" diff --git a/backend/migrations/20220905175719_go_lang.down.sql b/backend/migrations/20220905175719_go_lang.down.sql new file mode 100644 index 0000000000..d2f607c5b8 --- /dev/null +++ b/backend/migrations/20220905175719_go_lang.down.sql @@ -0,0 +1 @@ +-- Add down migration script here diff --git a/backend/migrations/20220905175719_go_lang.up.sql b/backend/migrations/20220905175719_go_lang.up.sql new file mode 100644 index 0000000000..9ac322b492 --- /dev/null +++ b/backend/migrations/20220905175719_go_lang.up.sql @@ -0,0 +1,2 @@ +-- Add up migration script here +ALTER TYPE SCRIPT_LANG ADD VALUE 'go'; \ No newline at end of file diff --git a/backend/openapi.yaml b/backend/openapi.yaml index b6d05097fa..6c2a43da9a 100644 --- a/backend/openapi.yaml +++ b/backend/openapi.yaml @@ -1742,7 +1742,7 @@ paths: type: string language: type: string - enum: [python3, deno] + enum: [python3, deno, go] kind: type: string enum: [script, failure, trigger, command] @@ -1802,6 +1802,27 @@ paths: schema: $ref: "#/components/schemas/MainArgSignature" + /scripts/go/tojsonschema: + post: + summary: inspect go code to infer jsonschema of arguments + operationId: goToJsonschema + tags: + - script + requestBody: + description: go code with the main function + required: true + content: + application/json: + schema: + type: string + responses: + "200": + description: parsed args + content: + application/json: + schema: + $ref: "#/components/schemas/MainArgSignature" + /w/{workspace}/scripts/archive/p/{path}: post: summary: archive script by path @@ -3240,7 +3261,7 @@ components: type: string language: type: string - enum: [python3, deno] + enum: [python3, deno, go] kind: type: string enum: [script, failure, trigger, command] @@ -3331,7 +3352,7 @@ components: type: boolean language: type: string - enum: [python3, deno] + enum: [python3, deno, go] required: - id - running @@ -3409,7 +3430,7 @@ components: type: boolean language: type: string - enum: [python3, deno] + enum: [python3, deno, go] is_skipped: type: boolean required: @@ -3781,7 +3802,7 @@ components: $ref: "#/components/schemas/ScriptArgs" language: type: string - enum: [python3, deno] + enum: [python3, deno, go] required: - content diff --git a/backend/src/jobs.rs b/backend/src/jobs.rs index 15d072ce4b..080c4788c6 100644 --- a/backend/src/jobs.rs +++ b/backend/src/jobs.rs @@ -1100,7 +1100,7 @@ pub enum JobPayload { ScriptHub { path: String }, ScriptHash { hash: ScriptHash, path: String }, Code(RawCode), - Dependencies { hash: ScriptHash, dependencies: Vec }, + Dependencies { hash: ScriptHash, dependencies: String, language: ScriptLang }, Flow(String), RawFlow { value: FlowValue, path: Option }, } @@ -1240,13 +1240,13 @@ pub async fn push<'c>( None, Some(language), ), - JobPayload::Dependencies { hash, dependencies } => ( + JobPayload::Dependencies { hash, dependencies, language } => ( Some(hash.0), None, - Some(dependencies.join("\n")), + Some(dependencies), JobKind::Dependencies, None, - Some(ScriptLang::Python3), + Some(language), ), JobPayload::RawFlow { value, path } => { (None, path, None, JobKind::FlowPreview, Some(value), None) diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 46f14e57e2..becf40f861 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -37,6 +37,10 @@ mod js_eval; mod more_serde; mod oauth2; mod parser; +mod parser_go; +mod parser_go_ast; +mod parser_go_scanner; +mod parser_go_token; mod parser_py; mod parser_ts; mod resources; diff --git a/backend/src/main.rs b/backend/src/main.rs index acef330d5c..714f7899d0 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -89,11 +89,15 @@ async fn main() -> anyhow::Result<()> { .ok() .and_then(|x| x.parse::().ok()) .unwrap_or(false); + let keep_job_dir = std::env::var("KEEP_JOB_DIR") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false); tracing::info!( "DISABLE_NSJAIL: {disable_nsjail}, DISABLE_NUSER: {disable_nuser}, BASE_URL: \ {base_url}, SLEEP_QUEUE: {sleep_queue}, NUM_WORKERS: {num_workers}, TIMEOUT: \ - {timeout}" + {timeout}, KEEP_JOB_DIR: {keep_job_dir}" ); windmill::run_workers( db.clone(), @@ -101,7 +105,13 @@ async fn main() -> anyhow::Result<()> { timeout, num_workers, sleep_queue, - WorkerConfig { disable_nsjail, disable_nuser, base_internal_url, base_url }, + WorkerConfig { + disable_nsjail, + disable_nuser, + base_internal_url, + base_url, + keep_job_dir, + }, rx.resubscribe(), ) .await?; diff --git a/backend/src/parser.rs b/backend/src/parser.rs index cc47d9cf53..8a1318104d 100644 --- a/backend/src/parser.rs +++ b/backend/src/parser.rs @@ -42,6 +42,7 @@ pub enum Typ { #[derive(Serialize, Clone, Debug, PartialEq)] pub struct Arg { pub name: String, + pub otyp: Option, pub typ: Typ, pub default: Option, pub has_default: bool, diff --git a/backend/src/parser_go.rs b/backend/src/parser_go.rs new file mode 100644 index 0000000000..e51b967bb6 --- /dev/null +++ b/backend/src/parser_go.rs @@ -0,0 +1,1281 @@ +#![allow(non_snake_case)] // TODO: switch to parse_* function naming + +use itertools::Itertools; + +use crate::error::to_anyhow; +use crate::parser::{Arg, MainArgSignature, Typ}; +use crate::parser_go_ast::{self, Ident}; +use crate::parser_go_ast::{Decl, Expr}; +use crate::parser_go_scanner; +use crate::parser_go_token::{Position, Token}; +use std::fmt; + +pub fn parse_go_sig(code: &str) -> crate::error::Result { + let filtered_code = filter_non_main(code); + let file = parse_file("main.go", &filtered_code).map_err(to_anyhow)?; + if let Some(Decl::FuncDecl(func)) = file.decls.first() { + let args = func + .type_ + .params + .list + .iter() + .map(|param| { + let (otyp, typ) = match ¶m.type_ { + Some(typ) => parse_go_typ(typ), + None => (None, Typ::Unknown), + }; + Arg { + name: param + .names + .as_ref() + .and_then(|x| x.first().map(|y| y.name.to_string())) + .unwrap_or_else(|| "".to_string()), + otyp, + typ, + default: None, + has_default: false, + } + }) + .collect_vec(); + Ok(MainArgSignature { star_args: false, star_kwargs: false, args }) + } else { + Err(crate::error::Error::BadRequest( + "no main function found".to_string(), + )) + } +} + +fn parse_go_typ(typ: &parser_go_ast::Expr) -> (Option, Typ) { + match typ { + Expr::Ident(Ident { name, .. }) => ( + Some((*name).to_string()), + match *name { + "int" => Typ::Int, + "string" => Typ::Str(None), + "bool" => Typ::Bool, + _ => Typ::Unknown, + }, + ), + Expr::ArrayType(array_type) => { + let (inner_otyp, inner_typ) = parse_go_typ(&*array_type.elt); + ( + inner_otyp.map(|x| format!("[]{x}")), + Typ::List(Box::new(inner_typ)), + ) + } + _ => (None, Typ::Unknown), + } +} + +#[cfg(test)] +mod tests { + + use crate::parser::{Arg, MainArgSignature, Typ}; + + use super::*; + + #[test] + fn test_parse_go_sig() -> anyhow::Result<()> { + let code = r#" + +package main + +import "fmt" + +func main(x int, y string, z bool, l []string) { + fmt.Println("hello world") +} + +"#; + //println!("{}", serde_json::to_string()?); + assert_eq!( + parse_go_sig(code)?, + MainArgSignature { + star_args: false, + star_kwargs: false, + args: vec![ + Arg { + otyp: Some("int".to_string()), + name: "x".to_string(), + typ: Typ::Int, + has_default: false, + default: None + }, + Arg { + otyp: Some("string".to_string()), + name: "y".to_string(), + typ: Typ::Str(None), + default: None, + has_default: false + }, + Arg { + otyp: Some("bool".to_string()), + name: "z".to_string(), + typ: Typ::Bool, + default: None, + has_default: false + }, + Arg { + otyp: Some("[]string".to_string()), + name: "l".to_string(), + typ: Typ::List(Box::new(Typ::Str(None))), + default: None, + has_default: false + }, + ] + } + ); + + Ok(()) + } +} + +fn filter_non_main(code: &str) -> String { + const FUNC_MAIN: &str = "func main("; + + let mut filtered_code = String::new(); + let mut code_iter = code.split("\n"); + let mut remaining: String = String::new(); + while let Some(line) = code_iter.next() { + if line.starts_with(FUNC_MAIN) { + filtered_code += FUNC_MAIN; + remaining += line.strip_prefix(FUNC_MAIN).unwrap(); + remaining += &code_iter.join("\n"); + break; + } + } + if filtered_code.is_empty() { + return String::new(); + } + let mut chars = remaining.chars(); + let mut open_parens = 1; + + while let Some(c) = chars.next() { + if c == '(' { + open_parens += 1; + } else if c == ')' { + open_parens -= 1; + } + filtered_code.push(c); + if open_parens == 0 { + break; + } + } + + filtered_code.push_str("{}"); + return filtered_code; +} + +#[derive(Debug)] +pub enum ParserError { + ScannerError(parser_go_scanner::ScannerError), + UnexpectedEndOfFile, + UnexpectedToken, + UnexpectedTokenAt { at: String, token: Token, literal: String }, +} + +impl std::error::Error for ParserError {} + +impl From for ParserError { + fn from(e: parser_go_scanner::ScannerError) -> Self { + Self::ScannerError(e) + } +} + +impl fmt::Display for ParserError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "parser error: {:?}", self) + } +} + +pub type Result = std::result::Result; + +trait ResultExt { + fn required(self) -> Result; +} + +impl ResultExt for Result> { + fn required(self) -> Result { + self.and_then(|node| node.map_or(Err(ParserError::UnexpectedToken), |node| Ok(node))) + } +} + +pub fn parse_file<'a>(filename: &'a str, buffer: &'a str) -> Result> { + let parser_go_scanner = parser_go_scanner::Scanner::new(filename, buffer); + let mut parser = Parser::new(parser_go_scanner); + parser.next()?; + parser.SourceFile().required().map_err(|err| match err { + ParserError::UnexpectedToken => ParserError::UnexpectedTokenAt { + at: parser.current_step.0.to_string(), + token: parser.current_step.1, + literal: parser.current_step.2.to_owned(), + }, + err => err, + }) +} + +struct Parser<'parser_go_scanner> { + steps: parser_go_scanner::IntoIter<'parser_go_scanner>, + current_step: parser_go_scanner::Step<'parser_go_scanner>, + expr_level: isize, +} + +impl<'parser_go_scanner> Parser<'parser_go_scanner> { + pub fn new(parser_go_scanner: parser_go_scanner::Scanner<'parser_go_scanner>) -> Self { + Self { + steps: parser_go_scanner.into_iter(), + current_step: (Position::default(), Token::EOF, ""), + expr_level: -1, + } + } + + // SourceFile = PackageClause ";" { ImportDecl ";" } { TopLevelDecl ";" } . + fn SourceFile(&mut self) -> Result>> { + let mut out = parser_go_ast::File { decls: vec![] }; + + while let Some(top_level_decl) = self.TopLevelDecl()? { + self.token(Token::SEMICOLON).required()?; + out.decls.push(top_level_decl); + } + + self.token(Token::EOF).required()?; + + Ok(Some(out)) + } + + // TopLevelDecl = Declaration | FunctionDecl | MethodDecl . + fn TopLevelDecl(&mut self) -> Result>> { + use Token::*; + Ok(match self.current_step.1 { + FUNC => Some(parser_go_ast::Decl::FuncDecl( + self.FunctionDecl_or_MethodDecl().required()?, + )), + _ => None, + }) + } + + // IdentifierList = identifier { "," identifier } . + fn IdentifierList(&mut self) -> Result>>> { + let mut out = match self.identifier()? { + Some(v) => vec![v], + None => return Ok(None), + }; + + while self.token(Token::COMMA)?.is_some() { + out.push(self.identifier().required()?); + } + + Ok(Some(out)) + } + + // ExpressionList = Expression { "," Expression } . + fn ExpressionList(&mut self) -> Result>>> { + let mut out = match self.Expression()? { + Some(v) => vec![v], + None => return Ok(None), + }; + + while self.token(Token::COMMA)?.is_some() { + out.push(self.Expression().required()?); + } + + Ok(Some(out)) + } + + // Expression = UnaryExpr | Expression binary_op Expression . + fn Expression(&mut self) -> Result>> { + let unary_expr = match self.UnaryExpr()? { + Some(v) => v, + None => return Ok(None), + }; + + self.expression(unary_expr, Token::lowest_precedence()) + } + + // https://en.wikipedia.org/wiki/Operator-precedence_parser + fn expression( + &mut self, + mut lhs: parser_go_ast::Expr<'parser_go_scanner>, + min_precedence: u8, + ) -> Result>> { + while let Some(op) = self.get_binary_op(min_precedence)? { + self.next()?; + + let mut rhs = self.UnaryExpr().required()?; + while self.get_binary_op(op.1.precedence() + 1)?.is_some() { + rhs = self.expression(rhs, op.1.precedence() + 1).required()?; + } + + lhs = parser_go_ast::Expr::BinaryExpr(parser_go_ast::BinaryExpr { + x: Box::new(lhs), + op_pos: op.0, + op: op.1, + y: Box::new(rhs), + }); + } + + Ok(Some(lhs)) + } + + // UnaryExpr = PrimaryExpr | unary_op UnaryExpr . + fn UnaryExpr(&mut self) -> Result>> { + if let Some(op) = self.unary_op()? { + let x = Box::new(self.UnaryExpr().required()?); + let expr = if op.1 == Token::MUL { + parser_go_ast::Expr::StarExpr(parser_go_ast::StarExpr { star: op.0, x }) + } else { + parser_go_ast::Expr::UnaryExpr(parser_go_ast::UnaryExpr { + op: op.1, + op_pos: op.0, + x, + }) + }; + return Ok(Some(expr)); + } + + self.PrimaryExpr() + } + + // PrimaryExpr = + // Operand | + // Conversion | + // MethodExpr | + // PrimaryExpr Selector | + // PrimaryExpr Index | + // PrimaryExpr Slice | + // PrimaryExpr TypeAssertion | + // PrimaryExpr Arguments . + fn PrimaryExpr(&mut self) -> Result>> { + let mut primary_expr = match self.Operand()? { + Some(v) => v, + None => return Ok(None), + }; + + loop { + match self.current_step.1 { + Token::PERIOD => { + primary_expr = self.Selector_or_TypeAssertion(primary_expr).required()?; + } + Token::LBRACK => { + primary_expr = self.Index_or_Slice(primary_expr).required()?; + } + Token::LPAREN => { + primary_expr = self.Arguments(primary_expr).required()?; + } + Token::LBRACE if self.expr_level >= 0 => { + unimplemented!("composite literal"); + } + _ => break, + } + } + + Ok(Some(primary_expr)) + } + + // Selector = "." identifier . + // TypeAssertion = "." "(" Type ")" . + fn Selector_or_TypeAssertion( + &mut self, + x: parser_go_ast::Expr<'parser_go_scanner>, + ) -> Result>> { + if self.token(Token::PERIOD)?.is_none() { + return Ok(None); + } + + if let Some(lparen) = self.token(Token::LPAREN)? { + let type_ = self.Type().required()?; + let rparen = self.token(Token::RPAREN).required()?; + return Ok(Some(parser_go_ast::Expr::TypeAssertExpr( + parser_go_ast::TypeAssertExpr { + x: Box::new(x), + lparen: lparen.0, + type_: Box::new(type_), + rparen: rparen.0, + }, + ))); + } + + Ok(Some(parser_go_ast::Expr::SelectorExpr( + parser_go_ast::SelectorExpr { x: Box::new(x), sel: self.identifier().required()? }, + ))) + } + + // Index = "[" Expression "]" . + // Slice = "[" [ Expression ] ":" [ Expression ] "]" | + // "[" [ Expression ] ":" Expression ":" Expression "]" . + fn Index_or_Slice( + &mut self, + x: parser_go_ast::Expr<'parser_go_scanner>, + ) -> Result>> { + let lbrack = match self.token(Token::LBRACK)? { + Some(v) => v, + None => return Ok(None), + }; + + let low = if let Some(low) = self.Expression()? { + if let Some(rbrack) = self.token(Token::RBRACK)? { + return Ok(Some(parser_go_ast::Expr::IndexExpr( + parser_go_ast::IndexExpr { + x: Box::new(x), + lbrack: lbrack.0, + index: Box::new(low), + rbrack: rbrack.0, + }, + ))); + } + Some(low) + } else { + None + }; + + self.token(Token::COLON).required()?; + + let high = if let Some(high) = self.Expression()? { + if self.token(Token::COLON)?.is_some() { + let max = self.Expression().required()?; + let rbrack = self.token(Token::RBRACK).required()?; + return Ok(Some(parser_go_ast::Expr::SliceExpr( + parser_go_ast::SliceExpr { + x: Box::new(x), + lbrack: lbrack.0, + low: low.map(Box::new), + high: Some(Box::new(high)), + max: Some(Box::new(max)), + slice3: true, + rbrack: rbrack.0, + }, + ))); + } + Some(high) + } else { + None + }; + let rbrack = self.token(Token::RBRACK).required()?; + + Ok(Some(parser_go_ast::Expr::SliceExpr( + parser_go_ast::SliceExpr { + x: Box::new(x), + lbrack: lbrack.0, + low: low.map(Box::new), + high: high.map(Box::new), + max: None, + slice3: false, + rbrack: rbrack.0, + }, + ))) + } + + // Arguments = "(" [ ( ExpressionList | Type [ "," ExpressionList ] ) [ "..." ] [ "," ] ] ")" . + fn Arguments( + &mut self, + x: parser_go_ast::Expr<'parser_go_scanner>, + ) -> Result>> { + let lparen = match self.token(Token::LPAREN)? { + Some(v) => v, + None => return Ok(None), + }; + + let mut args = if let Some(exprs) = self.ExpressionList()? { + exprs + } else if let Some(type_) = self.Type()? { + vec![type_] + } else { + vec![] + }; + + if self.token(Token::COMMA)?.is_some() { + let mut exprs = self.ExpressionList().required()?; + args.append(&mut exprs); + } + + let ellipsis = if !args.is_empty() { + let ellipsis = self.token(Token::ELLIPSIS)?; + self.token(Token::COMMA)?; + ellipsis + } else { + None + }; + + let rparen = self.token(Token::RPAREN).required()?; + + Ok(Some(parser_go_ast::Expr::CallExpr( + parser_go_ast::CallExpr { + fun: Box::new(x), + lparen: lparen.0, + args: Some(args), + ellipsis: ellipsis.map(|(pos, _, _)| pos), + rparen: rparen.0, + }, + ))) + } + + // Operand = Literal | OperandName | "(" Expression ")" . + // Literal = BasicLit | CompositeLit | FunctionLit . + // OperandName = identifier | QualifiedIdent . + fn Operand(&mut self) -> Result>> { + use Token::*; + Ok(match self.current_step.1 { + IDENT => Some(parser_go_ast::Expr::Ident(self.identifier().required()?)), + INT | FLOAT | IMAG | CHAR | STRING => { + Some(parser_go_ast::Expr::BasicLit(self.BasicLit().required()?)) + } + LPAREN => { + let lparen = self.token(Token::LPAREN).required()?; + let expr = self.Expression().required()?; + let rparen = self.token(Token::RPAREN).required()?; + return Ok(Some(parser_go_ast::Expr::ParenExpr( + parser_go_ast::ParenExpr { + lparen: lparen.0, + x: Box::new(expr), + rparen: rparen.0, + }, + ))); + } + FUNC => Some(parser_go_ast::Expr::FuncLit(self.FunctionLit().required()?)), + _ => self.CompositeLit()?.map(parser_go_ast::Expr::CompositeLit), + }) + } + + // CompositeLit = LiteralType LiteralValue . + // LiteralValue = "{" [ ElementList [ "," ] ] "}" . + // ElementList = KeyedElement { "," KeyedElement } . + fn CompositeLit(&mut self) -> Result>> { + let type_ = match self.LiteralType()? { + Some(v) => v, + None => return Ok(None), + }; + + let lbrace = self.token(Token::LBRACE).required()?; + + let mut elts = self.KeyedElement()?.map(|elt| vec![elt]); + if let Some(elts) = elts.as_mut() { + while self.token(Token::COMMA)?.is_some() { + if let Some(k) = self.KeyedElement()? { + elts.push(k); + } else { + break; + } + } + } + + let rbrace = self.token(Token::RBRACE).required()?; + + Ok(Some(parser_go_ast::CompositeLit { + type_: Box::new(type_), + lbrace: lbrace.0, + elts, + rbrace: rbrace.0, + incomplete: false, + })) + } + + // LiteralType = StructType | ArrayType | "[" "..." "]" ElementType | + // SliceType | MapType | TypeName . + fn LiteralType(&mut self) -> Result>> { + Ok(match self.current_step.1 { + Token::STRUCT => Some(parser_go_ast::Expr::StructType( + self.StructType().required()?, + )), + Token::LBRACK => Some(parser_go_ast::Expr::ArrayType( + self.ArrayType_or_SliceType::().required()?, + )), + Token::MAP => Some(parser_go_ast::Expr::MapType(self.MapType().required()?)), + Token::IDENT => Some(self.TypeName().required()?), + _ => None, + }) + } + + // KeyedElement = [ Key ":" ] Element . + // Key = FieldName | Expression | LiteralValue . + // FieldName = identifier . + // Element = Expression | LiteralValue . + fn KeyedElement(&mut self) -> Result>> { + let key = match self.Expression()? { + Some(v) => v, + None => return Ok(None), + }; + + if let Some(colon) = self.token(Token::COLON)? { + let value = self.Expression().required()?; + return Ok(Some(parser_go_ast::Expr::KeyValueExpr( + parser_go_ast::KeyValueExpr { + key: Box::new(key), + colon: colon.0, + value: Box::new(value), + }, + ))); + } + + Ok(Some(key)) + } + + // FunctionLit = "func" Signature FunctionBody . + fn FunctionLit(&mut self) -> Result>> { + let func = match self.token(Token::FUNC)? { + Some(v) => v, + None => return Ok(None), + }; + let type_ = self.Signature(Some(func.0)).required()?; + let body = self.FunctionBody().required()?; + + Ok(Some(parser_go_ast::FuncLit { type_, body })) + } + + // BasicLit = int_lit | float_lit | imaginary_lit | rune_lit | string_lit . + fn BasicLit(&mut self) -> Result>> { + Ok(match self.current_step.1 { + Token::INT => Some(self.int_lit().required()?), + Token::FLOAT => Some(self.float_lit().required()?), + Token::IMAG => Some(self.imaginary_lit().required()?), + Token::CHAR => Some(self.rune_lit().required()?), + Token::STRING => Some(self.string_lit().required()?), + _ => None, + }) + } + + // Type = TypeName | TypeLit | "(" Type ")" . + fn Type(&mut self) -> Result>> { + if self.token(Token::LPAREN)?.is_some() { + let type_ = self.Type().required()?; + self.token(Token::RPAREN).required()?; + return Ok(Some(type_)); + } + + if let Some(type_name) = self.TypeName()? { + return Ok(Some(type_name)); + } + + if let Some(type_lit) = self.TypeLit()? { + return Ok(Some(type_lit)); + } + + Ok(None) + } + + // TypeName = identifier | QualifiedIdent . + fn TypeName(&mut self) -> Result>> { + self.identifier_or_QualifiedIdent() + } + + // TypeLit = ArrayType | StructType | PointerType | FunctionType | InterfaceType | + // SliceType | MapType | ChannelType . + fn TypeLit(&mut self) -> Result>> { + Ok(match self.current_step.1 { + Token::LBRACK => Some(parser_go_ast::Expr::ArrayType( + self.ArrayType_or_SliceType::().required()?, + )), + Token::STRUCT => Some(parser_go_ast::Expr::StructType( + self.StructType().required()?, + )), + Token::MUL => Some(parser_go_ast::Expr::StarExpr( + self.PointerType().required()?, + )), + // TODO: FunctionType + Token::INTERFACE => Some(parser_go_ast::Expr::InterfaceType( + self.InterfaceType().required()?, + )), + Token::MAP => Some(parser_go_ast::Expr::MapType(self.MapType().required()?)), + Token::CHAN => Some(parser_go_ast::Expr::ChanType( + self.ChannelType().required()?, + )), + _ => None, + }) + } + + // ArrayType = "[" ArrayLength "]" ElementType . + // ArrayLength = Expression . + // SliceType = "[" "]" ElementType . + fn ArrayType_or_SliceType( + &mut self, + ) -> Result>> { + let lbrack = match self.token(Token::LBRACK)? { + Some(v) => v, + None => return Ok(None), + }; + + let len = if ELLIPSIS { + if let Some(ellipsis) = self.token(Token::ELLIPSIS)? { + Some(parser_go_ast::Expr::Ellipsis(parser_go_ast::Ellipsis { + ellipsis: ellipsis.0, + elt: None, + })) + } else { + self.Expression()? + } + } else { + self.Expression()? + }; + + self.token(Token::RBRACK).required()?; + + let element_type = self.ElementType().required()?; + + Ok(Some(parser_go_ast::ArrayType { + lbrack: lbrack.0, + len: len.map(Box::new), + elt: Box::new(element_type), + })) + } + + // MapType = "map" "[" KeyType "]" ElementType . + fn MapType(&mut self) -> Result>> { + let map = match self.token(Token::MAP)? { + Some(v) => v, + None => return Ok(None), + }; + self.token(Token::LBRACK).required()?; + let key_type = self.KeyType().required()?; + self.token(Token::RBRACK).required()?; + let element_type = self.ElementType().required()?; + + Ok(Some(parser_go_ast::MapType { + map: map.0, + key: Box::new(key_type), + value: Box::new(element_type), + })) + } + + // KeyType = Type . + fn KeyType(&mut self) -> Result>> { + self.Type() + } + + // ChannelType = ( "chan" | "chan" "<-" | "<-" "chan" ) ElementType . + fn ChannelType(&mut self) -> Result>> { + if let Some(chan) = self.token(Token::CHAN)? { + if let Some(arrow) = self.token(Token::ARROW)? { + let value = Box::new(self.ElementType().required()?); + return Ok(Some(parser_go_ast::ChanType { + begin: chan.0, + arrow: Some(arrow.0), + dir: parser_go_ast::ChanDir::SEND as u8, + value, + })); + } + + let value = Box::new(self.ElementType().required()?); + return Ok(Some(parser_go_ast::ChanType { + begin: chan.0, + arrow: None, + dir: parser_go_ast::ChanDir::SEND as u8 | parser_go_ast::ChanDir::RECV as u8, + value, + })); + } + + if let Some(arrow) = self.token(Token::ARROW)? { + self.token(Token::CHAN).required()?; + let value = Box::new(self.ElementType().required()?); + return Ok(Some(parser_go_ast::ChanType { + begin: arrow.0, + arrow: None, + dir: parser_go_ast::ChanDir::RECV as u8, + value, + })); + } + + Ok(None) + } + + // ElementType = Type . + fn ElementType(&mut self) -> Result>> { + self.Type() + } + + // PointerType = "*" BaseType . + fn PointerType(&mut self) -> Result>> { + let star = match self.token(Token::MUL)? { + Some(v) => v, + None => return Ok(None), + }; + + let x = Box::new(self.BaseType().required()?); + Ok(Some(parser_go_ast::StarExpr { star: star.0, x })) + } + + // BaseType = Type . + fn BaseType(&mut self) -> Result>> { + self.Type() + } + + // InterfaceType = "interface" "{" { ( MethodSpec | InterfaceTypeName ) ";" } "}" . + // MethodSpec = MethodName Signature . + fn InterfaceType( + &mut self, + ) -> Result>> { + let interface = match self.token(Token::INTERFACE)? { + Some(v) => v, + None => return Ok(None), + }; + + let lbrace = self.token(Token::LBRACE).required()?; + + let mut fields = vec![]; + loop { + if let Some(method_spec) = self.MethodName()? { + if let Some(signature) = self.Signature(None)? { + self.token(Token::SEMICOLON).required()?; + fields.push(parser_go_ast::Field { + doc: None, + names: Some(vec![method_spec]), + type_: Some(parser_go_ast::Expr::FuncType(signature)), + tag: None, + comment: None, + }); + continue; + } + + fields.push(parser_go_ast::Field { + doc: None, + names: None, + type_: Some(parser_go_ast::Expr::Ident(method_spec)), + tag: None, + comment: None, + }); + if self.token(Token::SEMICOLON)?.is_none() { + break; + } + continue; + }; + + if let Some(interface_type_name) = self.InterfaceTypeName()? { + fields.push(parser_go_ast::Field { + doc: None, + names: None, + type_: Some(interface_type_name), + tag: None, + comment: None, + }); + if self.token(Token::SEMICOLON)?.is_none() { + break; + } + continue; + } + + break; + } + + let rbrace = self.token(Token::RBRACE).required()?; + + Ok(Some(parser_go_ast::InterfaceType { + interface: interface.0, + methods: Some(parser_go_ast::FieldList { + opening: Some(lbrace.0), + list: fields, + closing: Some(rbrace.0), + }), + incomplete: false, + })) + } + + // MethodName = identifier . + fn MethodName(&mut self) -> Result>> { + self.identifier() + } + + // InterfaceTypeName = TypeName . + fn InterfaceTypeName(&mut self) -> Result>> { + self.TypeName() + } + + // StructType = "struct" "{" { FieldDecl ";" } "}" . + fn StructType(&mut self) -> Result>> { + let struct_ = match self.token(Token::STRUCT)? { + Some(v) => v, + None => return Ok(None), + }; + + let lbrace = self.token(Token::LBRACE).required()?; + + let mut fields = vec![]; + while let Some(field_decl) = self.FieldDecl()? { + fields.push(field_decl); + if self.token(Token::SEMICOLON)?.is_none() { + break; + } + } + + let rbrace = self.token(Token::RBRACE).required()?; + + Ok(Some(parser_go_ast::StructType { + struct_: struct_.0, + fields: Some(parser_go_ast::FieldList { + opening: Some(lbrace.0), + list: fields, + closing: Some(rbrace.0), + }), + incomplete: false, + })) + } + + // FieldDecl = (IdentifierList Type | EmbeddedField) [ Tag ] . + // EmbeddedField = [ "*" ] TypeName . + fn FieldDecl(&mut self) -> Result>> { + if let Some(star) = self.token(Token::MUL)? { + let type_name = Box::new(self.TypeName().required()?); + let tag = self.Tag()?; + return Ok(Some(parser_go_ast::Field { + doc: None, + type_: Some(parser_go_ast::Expr::StarExpr(parser_go_ast::StarExpr { + star: star.0, + x: type_name, + })), + names: None, + tag, + comment: None, + })); + }; + + if let Some(names) = self.IdentifierList()? { + if let Some(type_) = self.Type()? { + let tag = self.Tag()?; + return Ok(Some(parser_go_ast::Field { + doc: None, + names: Some(names), + type_: Some(type_), + tag, + comment: None, + })); + } + + if names.len() == 1 { + let name = names.into_iter().next().unwrap(); + let tag = self.Tag()?; + return Ok(Some(parser_go_ast::Field { + doc: None, + type_: Some(parser_go_ast::Expr::Ident(name)), + names: None, + tag, + comment: None, + })); + } + + return Err(ParserError::UnexpectedToken); + } + + if let Some(type_) = self.TypeName()? { + let tag = self.Tag()?; + return Ok(Some(parser_go_ast::Field { + doc: None, + type_: Some(type_), + names: None, + tag, + comment: None, + })); + } + + Ok(None) + } + + // Tag = string_lit . + fn Tag(&mut self) -> Result>> { + self.string_lit() + } + + // Signature = Parameters [ Result ] . + fn Signature( + &mut self, + func: Option>, + ) -> Result>> { + let params = match self.Parameters()? { + Some(v) => v, + None => return Ok(None), + }; + let results = self.Result()?; + + Ok(Some(parser_go_ast::FuncType { func, params, results })) + } + + // Result = Parameters | Type . + fn Result(&mut self) -> Result>> { + if let Some(parameters) = self.Parameters()? { + Ok(Some(parameters)) + } else if let Some(type_) = self.Type()? { + Ok(Some(parser_go_ast::FieldList { + opening: None, + list: vec![parser_go_ast::Field { + doc: None, + names: None, + tag: None, + type_: Some(type_), + comment: None, + }], + closing: None, + })) + } else { + Ok(None) + } + } + + // Parameters = "(" [ ParameterList [ "," ] ] ")" . + fn Parameters(&mut self) -> Result>> { + let lparen = match self.token(Token::LPAREN)? { + Some(v) => v, + None => return Ok(None), + }; + let list = self + .ParameterList()? + .map(|list| { + let _ = self.token(Token::COMMA); + list + }) + .unwrap_or_default(); + let rparen = self.token(Token::RPAREN).required()?; + + Ok(Some(parser_go_ast::FieldList { + opening: Some(lparen.0), + list, + closing: Some(rparen.0), + })) + } + + // ParameterList = ParameterDecl { "," ParameterDecl } . + // ParameterDecl = [ IdentifierList ] [ "..." ] Type . + fn ParameterList(&mut self) -> Result>>> { + let idents = match self.IdentifierList()? { + Some(v) => v, + None => return Ok(None), + }; + let type_ = self.Type()?; + + // If no type can be found, then the idents are types, e.g.: (bool, bool) + if type_.is_none() { + return Ok(Some( + idents + .into_iter() + .map(|ident| parser_go_ast::Field { + doc: None, + names: None, + type_: Some(parser_go_ast::Expr::Ident(ident)), + tag: None, + comment: None, + }) + .collect(), + )); + } + + // If a type can be found, then we expect idents + types: (a, b bool, c bool, d bool) + + let mut fields = vec![parser_go_ast::Field { + comment: None, + type_, + tag: None, + names: Some(idents), + doc: None, + }]; + + while self.token(Token::COMMA)?.is_some() { + let idents = self.IdentifierList().required()?; + let ellipsis = self.token(Token::ELLIPSIS)?; + let type_ = self.Type().required()?; + + if let Some(ellipsis) = ellipsis { + fields.push(parser_go_ast::Field { + comment: None, + type_: Some(parser_go_ast::Expr::Ellipsis(parser_go_ast::Ellipsis { + ellipsis: ellipsis.0, + elt: Some(Box::new(type_)), + })), + tag: None, + names: Some(idents), + doc: None, + }); + return Ok(Some(fields)); + } + + fields.push(parser_go_ast::Field { + comment: None, + type_: Some(type_), + tag: None, + names: Some(idents), + doc: None, + }); + } + + Ok(Some(fields)) + } + + // FunctionBody = Block . + fn FunctionBody(&mut self) -> Result>> { + self.Block() + } + + // Block = "{" StatementList "}" . + // StatementList = { Statement ";" } . + fn Block(&mut self) -> Result>> { + let lbrace = match self.token(Token::LBRACE)? { + Some(v) => v, + None => return Ok(None), + }; + + let list = vec![]; + + let rbrace = self.token(Token::RBRACE).required()?; + + Ok(Some(parser_go_ast::BlockStmt { + lbrace: lbrace.0, + list, + rbrace: rbrace.0, + })) + } + + // Receiver = Parameters . + fn Receiver(&mut self) -> Result>> { + self.Parameters() + } + + // identifier | QualifiedIdent + // QualifiedIdent = PackageName "." identifier . + // PackageName = identifier . + fn identifier_or_QualifiedIdent( + &mut self, + ) -> Result>> { + let ident = match self.identifier()? { + Some(v) => v, + None => return Ok(None), + }; + + if self.token(Token::PERIOD)?.is_some() { + let sel = self.identifier().required()?; + return Ok(Some(parser_go_ast::Expr::SelectorExpr( + parser_go_ast::SelectorExpr { x: Box::new(parser_go_ast::Expr::Ident(ident)), sel }, + ))); + } + + Ok(Some(parser_go_ast::Expr::Ident(ident))) + } + + // FunctionDecl | MethodDecl + // FunctionDecl = "func" FunctionName Signature [ FunctionBody ] . + // MethodDecl = "func" Receiver MethodName Signature [ FunctionBody ] . + // FunctionName = identifier . + // MethodName = identifier . + fn FunctionDecl_or_MethodDecl( + &mut self, + ) -> Result>> { + let func = match self.token(Token::FUNC)? { + Some(v) => v, + None => return Ok(None), + }; + let recv = self.Receiver()?; + let name = self.identifier().required()?; + let type_ = self.Signature(Some(func.0)).required()?; + let body = self.FunctionBody()?; + + Ok(Some(parser_go_ast::FuncDecl { + doc: None, + recv, + name, + type_, + body, + })) + } + + // unary_op = "+" | "-" | "!" | "^" | "*" | "&" | "<-" . + fn unary_op(&mut self) -> Result>> { + use Token::*; + Ok(match self.current_step { + step @ (_, ADD | SUB | NOT | MUL | XOR | AND | ARROW, _) => { + self.next()?; + Some(step) + } + _ => None, + }) + } + + // binary_op = "||" | "&&" | rel_op | add_op | mul_op . + // rel_op = "==" | "!=" | "<" | "<=" | ">" | ">=" . + // add_op = "+" | "-" | "|" | "^" . + // mul_op = "*" | "/" | "%" | "<<" | ">>" | "&" | "&^" . + fn get_binary_op( + &mut self, + min_precedence: u8, + ) -> Result>> { + use Token::*; + Ok(match self.current_step { + step @ (_, + /* binary_op */ + LOR | LAND | + /* rel_op */ + EQL | NEQ | LSS | LEQ | GTR | GEQ | + /* add_op */ + ADD | SUB | OR | XOR | + /* mul_op */ + MUL | QUO | REM | SHL | SHR | AND | AND_NOT + , _) if step.1.precedence() >= min_precedence => { + Some(step) + } + _ => None, + }) + } + + fn identifier(&mut self) -> Result>> { + self.token(Token::IDENT)? + .map_or(Ok(None), |(name_pos, _, name)| { + Ok(Some(parser_go_ast::Ident { name_pos, name, obj: None })) + }) + } + + fn int_lit(&mut self) -> Result>> { + self.token(Token::INT)? + .map_or(Ok(None), |(value_pos, kind, value)| { + Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) + }) + } + + fn float_lit(&mut self) -> Result>> { + self.token(Token::FLOAT)? + .map_or(Ok(None), |(value_pos, kind, value)| { + Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) + }) + } + + fn imaginary_lit(&mut self) -> Result>> { + self.token(Token::IMAG)? + .map_or(Ok(None), |(value_pos, kind, value)| { + Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) + }) + } + + fn rune_lit(&mut self) -> Result>> { + self.token(Token::CHAR)? + .map_or(Ok(None), |(value_pos, kind, value)| { + Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) + }) + } + + fn string_lit(&mut self) -> Result>> { + self.token(Token::STRING)? + .map_or(Ok(None), |(value_pos, kind, value)| { + Ok(Some(parser_go_ast::BasicLit { value_pos, kind, value })) + }) + } + + /// Returns the current step and advances to the next one, but only if it matches the expected + /// token. [`Parser::next`] is automatically called for you. + fn token( + &mut self, + expected: Token, + ) -> Result>> { + Ok(match self.current_step { + step @ (_, tok, _) if tok == expected => { + if expected != Token::EOF { + self.next()?; + } + Some(step) + } + _ => None, + }) + } + + /// Advances to the next token. Skips all the comment tokens. + fn next(&mut self) -> Result<()> { + if let Some(step) = self + .steps + .find(|step| !matches!(step, Ok((_, Token::COMMENT, _)))) + { + self.current_step = step?; + return Ok(()); + } + Err(ParserError::UnexpectedEndOfFile) + } +} diff --git a/backend/src/parser_go_ast.rs b/backend/src/parser_go_ast.rs new file mode 100644 index 0000000000..cf813f4dac --- /dev/null +++ b/backend/src/parser_go_ast.rs @@ -0,0 +1,347 @@ +#![allow(clippy::large_enum_variant)] // TODO: we allow large enum variant for now, let's profile properly to see if we want to box. + +use crate::parser_go_token::{Position, Token}; +use std::collections::BTreeMap; + +// https://pkg.go.dev/go/ast#CommentGroup +#[derive(Debug)] +pub struct CommentGroup { + // List []*Comment // len(List) > 0 +} + +// https://pkg.go.dev/go/ast#FieldList +#[derive(Debug)] +pub struct FieldList<'a> { + pub opening: Option>, // position of opening parenthesis/brace, if any + pub list: Vec>, // field list; or nil + pub closing: Option>, // position of closing parenthesis/brace, if any +} + +// https://pkg.go.dev/go/ast#Field +#[derive(Debug)] +pub struct Field<'a> { + pub doc: Option, // associated documentation; or nil + pub names: Option>>, // field/method/(type) parameter names, or type "type"; or nil + pub type_: Option>, // field/method/parameter type, type list type; or nil + pub tag: Option>, // field tag; or nil + pub comment: Option, // line comments; or nil +} + +// https://pkg.go.dev/go/ast#File +#[derive(Debug)] +pub struct File<'a> { + // package name + pub decls: Vec>, // top-level declarations; or nil // list of all comments in the source file +} + +// https://pkg.go.dev/go/ast#FuncDecl +#[derive(Debug)] +pub struct FuncDecl<'a> { + pub doc: Option, // associated documentation; or nil + pub recv: Option>, // receiver (methods); or nil (functions) + pub name: Ident<'a>, // function/method name + pub type_: FuncType<'a>, // function signature: type and value parameters, results, and position of "func" keyword + pub body: Option>, // function body; or nil for external (non-Go) function +} + +// https://pkg.go.dev/go/ast#BlockStmt +#[derive(Debug)] +pub struct BlockStmt<'a> { + pub lbrace: Position<'a>, // position of "{" + pub list: Vec, + pub rbrace: Position<'a>, // position of "}", if any (may be absent due to syntax error) +} + +// https://pkg.go.dev/go/ast#FuncType +#[derive(Debug)] +pub struct FuncType<'a> { + pub func: Option>, // position of "func" keyword (token.NoPos if there is no "func") + pub params: FieldList<'a>, // (incoming) parameters; non-nil + pub results: Option>, // (outgoing) results; or nil +} + +// https://pkg.go.dev/go/ast#Ident +#[derive(Debug)] +pub struct Ident<'a> { + pub name_pos: Position<'a>, // identifier position + pub name: &'a str, // identifier name + pub obj: Option>>, // denoted object; or nil +} + +// https://pkg.go.dev/go/ast#ValueSpec +#[derive(Debug)] +pub struct ValueSpec<'a> { + pub doc: Option, // associated documentation; or nil + pub names: Vec>, // value names (len(Names) > 0) + pub type_: Option>, // value type; or nil + pub values: Option>>, // initial values; or nil + pub comment: Option, // line comments; or nil +} + +// https://pkg.go.dev/go/ast#BasicLit +#[derive(Debug)] +pub struct BasicLit<'a> { + pub value_pos: Position<'a>, // literal position + pub kind: Token, // token.INT, token.FLOAT, token.IMAG, token.CHAR, or token.STRING + pub value: &'a str, // literal string; e.g. 42, 0x7f, 3.14, 1e-9, 2.4i, 'a', '\x7f', "foo" or `\m\n\o` +} + +// https://pkg.go.dev/go/ast#Object +#[derive(Debug)] +pub struct Object<'a> { + pub kind: ObjKind, + pub name: &'a str, // declared name + pub decl: Option, // corresponding Field, XxxSpec, FuncDecl, LabeledStmt, AssignStmt, Scope; or nil + pub data: Option, // object-specific data; or nil + pub type_: Option<()>, // placeholder for type information; may be nil +} + +// https://pkg.go.dev/go/ast#Ellipsis +#[derive(Debug)] +pub struct Ellipsis<'a> { + pub ellipsis: Position<'a>, // position of "..." + pub elt: Option>>, // ellipsis element type (parameter lists only); or nil +} + +// https://pkg.go.dev/go/ast#Ellipsis +#[derive(Debug)] +pub struct TypeAssertExpr<'a> { + pub x: Box>, // expression + pub lparen: Position<'a>, // position of "(" + pub type_: Box>, // asserted type; nil means type switch X.(type) + pub rparen: Position<'a>, // position of ")" +} + +// https://pkg.go.dev/go/ast#SliceExpr +#[derive(Debug)] +pub struct SliceExpr<'a> { + pub x: Box>, // expression + pub lbrack: Position<'a>, // position of "[" + pub low: Option>>, // begin of slice range; or nil + pub high: Option>>, // end of slice range; or nil + pub max: Option>>, // maximum capacity of slice; or nil + pub slice3: bool, // true if 3-index slice (2 colons present) + pub rbrack: Position<'a>, // position of "]" +} + +// https://pkg.go.dev/go/ast#ObjKind +#[derive(Debug)] +pub enum ObjKind {} + +#[derive(Debug)] +pub enum ObjDecl {} + +// https://pkg.go.dev/go/ast#Decl +#[derive(Debug)] +pub enum Decl<'a> { + FuncDecl(FuncDecl<'a>), +} + +// https://pkg.go.dev/go/ast#Scope +#[derive(Debug)] +pub struct Scope<'a> { + pub outer: Option>>, + pub objects: BTreeMap<&'a str, Object<'a>>, +} + +// https://pkg.go.dev/go/ast#GenDecl +#[derive(Debug)] +pub struct GenDecl<'a> { + pub doc: Option, // associated documentation; or nil + pub tok_pos: Position<'a>, // position of Tok + pub tok: Token, // IMPORT, CONST, TYPE, or VAR + pub lparen: Option>, // position of '(', if any + pub specs: Vec, + pub rparen: Option>, // position of ')', if any +} + +// https://pkg.go.dev/go/ast#AssignStmt +#[derive(Debug)] +pub struct AssignStmt<'a> { + pub lhs: Vec>, + pub tok_pos: Position<'a>, // position of Tok + pub tok: Token, // assignment token, DEFINE + pub rhs: Vec>, +} + +// https://pkg.go.dev/go/ast#BinaryExpr +#[derive(Debug)] +pub struct BinaryExpr<'a> { + pub x: Box>, // left operand + pub op_pos: Position<'a>, // position of Op + pub op: Token, // operator + pub y: Box>, // right operand +} + +// https://pkg.go.dev/go/ast#ReturnStmt +#[derive(Debug)] +pub struct ReturnStmt<'a> { + pub return_: Position<'a>, // position of "return" keyword + pub results: Vec>, // result expressions; or nil +} + +// https://pkg.go.dev/go/ast#TypeSpec +#[derive(Debug)] +pub struct TypeSpec<'a> { + pub doc: Option, // associated documentation; or nil + pub name: Option>, // type name + pub assign: Option>, // position of '=', if any + pub type_: Expr<'a>, // *Ident, *ParenExpr, *SelectorExpr, *StarExpr, or any of the *XxxTypes + pub comment: Option, // line comments; or nil +} + +// https://pkg.go.dev/go/ast#StructType +#[derive(Debug)] +pub struct StructType<'a> { + pub struct_: Position<'a>, // position of "struct" keyword + pub fields: Option>, // list of field declarations + pub incomplete: bool, // true if (source) fields are missing in the Fields list +} + +// https://pkg.go.dev/go/ast#StarExpr +#[derive(Debug)] +pub struct StarExpr<'a> { + pub star: Position<'a>, // position of "*" + pub x: Box>, // operand +} + +// https://pkg.go.dev/go/ast#InterfaceType +#[derive(Debug)] +pub struct InterfaceType<'a> { + pub interface: Position<'a>, // position of "interface" keyword + pub methods: Option>, // list of embedded interfaces, methods, or types + pub incomplete: bool, // true if (source) methods or types are missing in the Methods list +} + +// https://pkg.go.dev/go/ast#UnaryExpr +#[derive(Debug)] +pub struct UnaryExpr<'a> { + pub op_pos: Position<'a>, // position of Op + pub op: Token, // operator + pub x: Box>, // operand +} + +// https://pkg.go.dev/go/ast#CallExpr +#[derive(Debug)] +pub struct CallExpr<'a> { + pub fun: Box>, // function expression + pub lparen: Position<'a>, // position of "(" + pub args: Option>>, // function arguments; or nil + pub ellipsis: Option>, // position of "..." (token.NoPos if there is no "...") + pub rparen: Position<'a>, // position of ")" +} + +// https://pkg.go.dev/go/ast#SelectorExpr +#[derive(Debug)] +pub struct SelectorExpr<'a> { + pub x: Box>, // expression + pub sel: Ident<'a>, // field selector +} + +// https://pkg.go.dev/go/ast#ParenExpr +#[derive(Debug)] +pub struct ParenExpr<'a> { + pub lparen: Position<'a>, // position of "(" + pub x: Box>, // parenthesized expression + pub rparen: Position<'a>, // position of ")" +} + +// https://pkg.go.dev/go/ast#FuncLit +#[derive(Debug)] +pub struct FuncLit<'a> { + pub type_: FuncType<'a>, // function type + pub body: BlockStmt<'a>, // function body +} + +// https://pkg.go.dev/go/ast#ChanType +#[derive(Debug)] +pub struct ChanType<'a> { + pub begin: Position<'a>, // position of "chan" keyword or "<-" (whichever comes first) + pub arrow: Option>, // position of "<-" (token.NoPos if there is no "<-") + pub dir: u8, // channel direction + pub value: Box>, // value type +} + +// htt/opt/visual-studio-code/resources/app/out/vs/code/electron-sandbox/workbench/workbench.htmlps://pkg.go.dev/go/ast#IndexExpr +#[derive(Debug)] +pub struct IndexExpr<'a> { + pub x: Box>, // expression + pub lbrack: Position<'a>, // position of "[" + pub index: Box>, // index expression + pub rbrack: Position<'a>, // position of "]" +} + +// https://pkg.go.dev/go/ast#MapType +#[derive(Debug)] +pub struct MapType<'a> { + pub map: Position<'a>, + pub key: Box>, + pub value: Box>, +} + +// https://pkg.go.dev/go/ast#CompositeLit +#[derive(Debug)] +pub struct CompositeLit<'a> { + pub type_: Box>, // literal type; or nil + pub lbrace: Position<'a>, // position of "{" + pub elts: Option>>, // list of composite elements; or nil + pub rbrace: Position<'a>, // position of "}" + pub incomplete: bool, // true if (source) expressions are missing in the Elts list +} + +// https://pkg.go.dev/go/ast#KeyValueExpr +#[derive(Debug)] +pub struct KeyValueExpr<'a> { + pub key: Box>, + pub colon: Position<'a>, // position of ":" + pub value: Box>, +} + +// https://pkg.go.dev/go/ast#ArrayType +#[derive(Debug)] +pub struct ArrayType<'a> { + pub lbrack: Position<'a>, // position of "[" + pub len: Option>>, // Ellipsis node for [...]T array types, nil for slice types + pub elt: Box>, // element type +} + +// https://pkg.go.dev/go/ast#ChanDir +#[derive(Debug)] +pub enum ChanDir { + SEND = 1 << 0, + RECV = 1 << 1, +} + +// https://pkg.go.dev/go/ast#Spec +#[derive(Debug)] +pub enum Spec {} + +// https://pkg.go.dev/go/ast#Expr +#[derive(Debug)] +pub enum Expr<'a> { + ArrayType(ArrayType<'a>), + BasicLit(BasicLit<'a>), + BinaryExpr(BinaryExpr<'a>), + CallExpr(CallExpr<'a>), + ChanType(ChanType<'a>), + CompositeLit(CompositeLit<'a>), + Ellipsis(Ellipsis<'a>), + FuncLit(FuncLit<'a>), + FuncType(FuncType<'a>), + Ident(Ident<'a>), + IndexExpr(IndexExpr<'a>), + InterfaceType(InterfaceType<'a>), + KeyValueExpr(KeyValueExpr<'a>), + MapType(MapType<'a>), + ParenExpr(ParenExpr<'a>), + SelectorExpr(SelectorExpr<'a>), + SliceExpr(SliceExpr<'a>), + StarExpr(StarExpr<'a>), + StructType(StructType<'a>), + TypeAssertExpr(TypeAssertExpr<'a>), + UnaryExpr(UnaryExpr<'a>), +} + +// https://pkg.go.dev/go/ast#Stmt +#[derive(Debug)] +pub enum Stmt {} diff --git a/backend/src/parser_go_scanner.rs b/backend/src/parser_go_scanner.rs new file mode 100644 index 0000000000..840539fce6 --- /dev/null +++ b/backend/src/parser_go_scanner.rs @@ -0,0 +1,948 @@ +// https://golang.org/ref/spec#Lexical_elements + +use crate::parser_go_token::{Position, Token}; +use phf::{phf_map, Map}; +use std::fmt; +use unicode_general_category::{get_general_category, GeneralCategory}; + +pub type Step<'a> = (Position<'a>, Token, &'a str); + +#[derive(Debug)] +pub enum ScannerError { + HexadecimalNotFound, + OctalNotFound, + UnterminatedComment, + UnterminatedEscapedChar, + UnterminatedRune, + UnterminatedString, + InvalidDirective, +} + +impl std::error::Error for ScannerError {} + +impl fmt::Display for ScannerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "scanner error: {:?}", self) + } +} + +pub type Result = std::result::Result; + +#[derive(Debug)] +pub struct Scanner<'a> { + directory: &'a str, + file: &'a str, + buffer: &'a str, + // + chars: std::iter::Peekable>, + current_char: Option, + current_char_len: usize, + // + offset: usize, + line: usize, + column: usize, + start_offset: usize, + start_line: usize, + start_column: usize, + // + hide_column: bool, + insert_semi: bool, + pending_line_info: Option>, +} + +type LineInfo<'a> = (Option<&'a str>, usize, Option, bool); + +impl<'a> Scanner<'a> { + pub fn new(filename: &'a str, buffer: &'a str) -> Self { + let (directory, file) = filename.rsplit_once('/').unwrap_or(("", filename)); + let mut s = Scanner { + directory, + file, + buffer, + // + chars: buffer.chars().peekable(), + current_char: None, + current_char_len: 0, + // + offset: 0, + line: 1, + column: 1, + start_offset: 0, + start_line: 1, + start_column: 1, + // + hide_column: false, + insert_semi: false, + pending_line_info: None, + }; + s.next(); // read the first character + s + } + + #[allow(clippy::cognitive_complexity)] // Allow complex scan function + pub fn scan(&mut self) -> Result> { + let insert_semi = self.insert_semi; + self.insert_semi = false; + + while let Some(c) = self.current_char { + self.reset_start(); + + match c { + ' ' | '\t' | '\r' => { + self.next(); + } + + '\n' => { + self.next(); + if insert_semi { + return Ok((self.position(), Token::SEMICOLON, "\n")); + } + } + + _ => break, + } + } + + if let Some(c) = self.current_char { + match c { + '+' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::ADD_ASSIGN, "")); + } + Some('+') => { + self.insert_semi = true; + self.next(); + return Ok((self.position(), Token::INC, "")); + } + _ => return Ok((self.position(), Token::ADD, "")), + } + } + + '-' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::SUB_ASSIGN, "")); + } + Some('-') => { + self.insert_semi = true; + self.next(); + return Ok((self.position(), Token::DEC, "")); + } + _ => return Ok((self.position(), Token::SUB, "")), + } + } + + '*' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::MUL_ASSIGN, "")); + } + _ => return Ok((self.position(), Token::MUL, "")), + } + } + + '/' => match self.peek() { + Some('=') => { + self.next(); + self.next(); + return Ok((self.position(), Token::QUO_ASSIGN, "")); + } + Some('/') => { + if insert_semi { + return Ok((self.position(), Token::SEMICOLON, "\n")); + } + return self.scan_line_comment(); + } + Some('*') => { + if insert_semi && self.find_line_end() { + return Ok((self.position(), Token::SEMICOLON, "\n")); + } + return self.scan_general_comment(); + } + _ => { + self.next(); + return Ok((self.position(), Token::QUO, "")); + } + }, + + '%' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::REM_ASSIGN, "")); + } + _ => return Ok((self.position(), Token::REM, "")), + } + } + + '&' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::AND_ASSIGN, "")); + } + Some('&') => { + self.next(); + return Ok((self.position(), Token::LAND, "")); + } + Some('^') => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::AND_NOT_ASSIGN, "")); + } + _ => return Ok((self.position(), Token::AND_NOT, "")), + } + } + _ => return Ok((self.position(), Token::AND, "")), + } + } + + '|' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::OR_ASSIGN, "")); + } + Some('|') => { + self.next(); + return Ok((self.position(), Token::LOR, "")); + } + _ => return Ok((self.position(), Token::OR, "")), + } + } + + '^' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::XOR_ASSIGN, "")); + } + _ => return Ok((self.position(), Token::XOR, "")), + } + } + + '<' => { + self.next(); + match self.current_char { + Some('<') => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::SHL_ASSIGN, "")); + } + _ => return Ok((self.position(), Token::SHL, "")), + } + } + Some('=') => { + self.next(); + return Ok((self.position(), Token::LEQ, "")); + } + Some('-') => { + self.next(); + return Ok((self.position(), Token::ARROW, "")); + } + _ => return Ok((self.position(), Token::LSS, "")), + } + } + + '>' => { + self.next(); + match self.current_char { + Some('>') => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::SHR_ASSIGN, "")); + } + _ => { + return Ok((self.position(), Token::SHR, "")); + } + } + } + Some('=') => { + self.next(); + return Ok((self.position(), Token::GEQ, "")); + } + _ => return Ok((self.position(), Token::GTR, "")), + } + } + + ':' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::DEFINE, "")); + } + _ => return Ok((self.position(), Token::COLON, "")), + } + } + + '!' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::NEQ, "")); + } + _ => return Ok((self.position(), Token::NOT, "")), + } + } + + ',' => { + self.next(); + return Ok((self.position(), Token::COMMA, "")); + } + + '(' => { + self.next(); + return Ok((self.position(), Token::LPAREN, "")); + } + + ')' => { + self.insert_semi = true; + self.next(); + return Ok((self.position(), Token::RPAREN, "")); + } + + '[' => { + self.next(); + return Ok((self.position(), Token::LBRACK, "")); + } + + ']' => { + self.insert_semi = true; + self.next(); + return Ok((self.position(), Token::RBRACK, "")); + } + + '{' => { + self.next(); + return Ok((self.position(), Token::LBRACE, "")); + } + + '}' => { + self.insert_semi = true; + self.next(); + return Ok((self.position(), Token::RBRACE, "")); + } + + ';' => { + self.next(); + return Ok((self.position(), Token::SEMICOLON, ";")); + } + + '.' => { + self.next(); + match self.current_char { + Some('0'..='9') => return self.scan_int_or_float_or_imag(true), + Some('.') => match self.peek() { + Some('.') => { + self.next(); + self.next(); + return Ok((self.position(), Token::ELLIPSIS, "")); + } + _ => return Ok((self.position(), Token::PERIOD, "")), + }, + _ => return Ok((self.position(), Token::PERIOD, "")), + } + } + + '=' => { + self.next(); + match self.current_char { + Some('=') => { + self.next(); + return Ok((self.position(), Token::EQL, "")); + } + _ => return Ok((self.position(), Token::ASSIGN, "")), + } + } + + '0'..='9' => return self.scan_int_or_float_or_imag(false), + '\'' => return self.scan_rune(), + '"' => return self.scan_interpreted_string(), + '`' => return self.scan_raw_string(), + _ => return self.scan_pkg_or_keyword_or_ident(), + }; + } + + self.reset_start(); + if insert_semi { + Ok((self.position(), Token::SEMICOLON, "\n")) + } else { + Ok((self.position(), Token::EOF, "")) + } + } + + // https://golang.org/ref/spec#Keywords + // https://golang.org/ref/spec#Identifiers + fn scan_pkg_or_keyword_or_ident(&mut self) -> Result> { + self.next(); + + while let Some(c) = self.current_char { + if !(is_letter(c) || is_unicode_digit(c)) { + break; + } + self.next() + } + + let pos = self.position(); + let literal = self.literal(); + + if literal.len() > 1 { + if let Some(&token) = KEYWORDS.get(literal) { + self.insert_semi = matches!( + token, + Token::BREAK | Token::CONTINUE | Token::FALLTHROUGH | Token::RETURN + ); + return Ok((pos, token, literal)); + } + } + + self.insert_semi = true; + Ok((pos, Token::IDENT, literal)) + } + + // https://golang.org/ref/spec#Integer_literals + // https://golang.org/ref/spec#Floating-point_literals + // https://golang.org/ref/spec#Imaginary_literals + fn scan_int_or_float_or_imag(&mut self, preceding_dot: bool) -> Result> { + self.insert_semi = true; + + let mut token = Token::INT; + let mut digits = "_0123456789"; + let mut exp = "eE"; + + if !preceding_dot { + if matches!(self.current_char, Some('0')) { + self.next(); + match self.current_char { + Some('b' | 'B') => { + digits = "_01"; + exp = ""; + self.next(); + } + Some('o' | 'O') => { + digits = "_01234567"; + exp = ""; + self.next(); + } + Some('x' | 'X') => { + digits = "_0123456789abcdefABCDEF"; + exp = "pP"; + self.next(); + } + _ => {} + }; + } + + while let Some(c) = self.current_char { + if !digits.contains(c) { + break; + } + self.next(); + } + } + + if preceding_dot || matches!(self.current_char, Some('.')) { + token = Token::FLOAT; + self.next(); + while let Some(c) = self.current_char { + if !digits.contains(c) { + break; + } + self.next(); + } + } + + if !exp.is_empty() { + if let Some(c) = self.current_char { + if exp.contains(c) { + token = Token::FLOAT; + self.next(); + if matches!(self.current_char, Some('-' | '+')) { + self.next(); + } + while let Some(c) = self.current_char { + if !matches!(c, '_' | '0'..='9') { + break; + } + self.next(); + } + } + } + } + + if matches!(self.current_char, Some('i')) { + token = Token::IMAG; + self.next(); + } + + Ok((self.position(), token, self.literal())) + } + + // https://golang.org/ref/spec#Rune_literals + fn scan_rune(&mut self) -> Result> { + self.insert_semi = true; + self.next(); + + match self.current_char { + Some('\\') => self.require_escaped_char::<'\''>()?, + Some(_) => self.next(), + _ => return Err(ScannerError::UnterminatedRune), + } + + if matches!(self.current_char, Some('\'')) { + self.next(); + return Ok((self.position(), Token::CHAR, self.literal())); + } + + Err(ScannerError::UnterminatedRune) + } + + // https://golang.org/ref/spec#String_literals + fn scan_interpreted_string(&mut self) -> Result> { + self.insert_semi = true; + self.next(); + + while let Some(c) = self.current_char { + match c { + '"' => { + self.next(); + return Ok((self.position(), Token::STRING, self.literal())); + } + '\\' => self.require_escaped_char::<'"'>()?, + _ => self.next(), + } + } + + Err(ScannerError::UnterminatedString) + } + + // https://golang.org/ref/spec#String_literals + fn scan_raw_string(&mut self) -> Result> { + self.insert_semi = true; + self.next(); + + while let Some(c) = self.current_char { + match c { + '`' => { + self.next(); + return Ok((self.position(), Token::STRING, self.literal())); + } + _ => self.next(), + } + } + + Err(ScannerError::UnterminatedString) + } + + // https://golang.org/ref/spec#Comments + fn scan_general_comment(&mut self) -> Result> { + self.next(); + self.next(); + + while let Some(c) = self.current_char { + match c { + '*' => { + self.next(); + if matches!(self.current_char, Some('/')) { + self.next(); + + let pos = self.position(); + let lit = self.literal(); + + // look for compiler directives + self.directive(&lit["/*".len()..lit.len() - "*/".len()], true)?; + + return Ok((pos, Token::COMMENT, lit)); + } + } + _ => self.next(), + } + } + + Err(ScannerError::UnterminatedComment) + } + + // https://golang.org/ref/spec#Comments + fn scan_line_comment(&mut self) -> Result> { + self.next(); + self.next(); + + while let Some(c) = self.current_char { + if is_newline(c) { + break; + } + self.next(); + } + + let pos = self.position(); + let lit = self.literal(); + + // look for compiler directives (at the beginning of line) + if self.start_column == 1 { + self.directive(lit["//".len()..].trim_end(), false)?; + } + + Ok((pos, Token::COMMENT, self.literal())) + } + + // https://pkg.go.dev/cmd/compile#hdr-Compiler_Directives + fn directive(&mut self, input: &'a str, immediate: bool) -> Result<()> { + if let Some(line_directive) = input.strip_prefix("line ") { + self.pending_line_info = self.parse_line_directive(line_directive)?; + if immediate { + self.consume_pending_line_info(); + } + } + Ok(()) + } + + fn parse_line_directive(&mut self, line_directive: &'a str) -> Result>> { + if let Some((file, line)) = line_directive.rsplit_once(':') { + let line = line.parse().map_err(|_| ScannerError::InvalidDirective)?; + + if let Some((file, l)) = file.rsplit_once(':') { + if let Ok(l) = l.parse() { + //line :line:col + //line filename:line:col + /*line :line:col*/ + /*line filename:line:col*/ + let file = if !file.is_empty() { Some(file) } else { None }; + let col = Some(line); + let line = l; + let hide_column = false; + return Ok(Some((file, line, col, hide_column))); + } + } + + //line :line + //line filename:line + /*line :line*/ + /*line filename:line*/ + Ok(Some((Some(file), line, None, true))) + } else { + Ok(None) + } + } + + const fn find_line_end(&self) -> bool { + let buffer = self.buffer.as_bytes(); + let mut in_comment = true; + + let mut i = self.offset; + let max = self.buffer.len(); + while i < max { + let c = buffer[i] as char; + + if i < max - 1 { + let n = buffer[i + 1] as char; + + if !in_comment && c == '/' && n == '/' { + return true; + } + + if c == '/' && n == '*' { + i += 2; + in_comment = true; + continue; + } + + if c == '*' && n == '/' { + i += 2; + in_comment = false; + continue; + } + } + + if is_newline(c) { + return true; + } + + if !in_comment && !matches!(c, ' ' | '\t' | '\r') { + return false; + } + + i += 1; + } + + !in_comment + } + + fn consume_pending_line_info(&mut self) { + if let Some(line_info) = self.pending_line_info.take() { + if let Some(file) = line_info.0 { + self.file = file; + } + + self.line = line_info.1; + + if let Some(column) = line_info.2 { + self.column = column; + } + + self.hide_column = line_info.3; + } + } + + fn peek(&mut self) -> Option { + self.chars.peek().copied() + } + + fn next(&mut self) { + self.offset += self.current_char_len; + self.column += self.current_char_len; + let last_char = self.current_char; + + self.current_char = self.chars.next(); + if let Some(c) = self.current_char { + self.current_char_len = c.len_utf8(); + if matches!(last_char, Some('\n')) { + self.line += 1; + self.column = 1; + self.consume_pending_line_info(); + } + } else { + self.current_char_len = 0 + } + } + + const fn position(&self) -> Position<'a> { + Position { + directory: self.directory, + file: self.file, + offset: self.start_offset, + line: self.start_line, + column: if self.hide_column { + 0 + } else { + self.start_column + }, + } + } + + fn reset_start(&mut self) { + self.start_offset = self.offset; + self.start_line = self.line; + self.start_column = self.column; + } + + fn literal(&self) -> &'a str { + &self.buffer[self.start_offset..self.offset] + } + + fn require_escaped_char(&mut self) -> Result<()> { + self.next(); + + let c = self + .current_char + .ok_or(ScannerError::UnterminatedEscapedChar)?; + + // TODO: move this to the match when const generics can be referenced in patterns + if c == DELIM { + self.next(); + return Ok(()); + } + + match c { + 'a' | 'b' | 'f' | 'n' | 'r' | 't' | 'v' | '\\' => self.next(), + 'x' => { + self.next(); + self.require_hex_digits::<2>()? + } + 'u' => { + self.next(); + self.require_hex_digits::<4>()?; + } + 'U' => { + self.next(); + self.require_hex_digits::<8>()?; + } + '0'..='7' => self.require_octal_digits::<3>()?, + _ => return Err(ScannerError::UnterminatedEscapedChar), + } + + Ok(()) + } + + fn require_octal_digits(&mut self) -> Result<()> { + for _ in 0..COUNT { + let c = self.current_char.ok_or(ScannerError::OctalNotFound)?; + + if !is_octal_digit(c) { + return Err(ScannerError::OctalNotFound); + } + + self.next(); + } + + Ok(()) + } + + fn require_hex_digits(&mut self) -> Result<()> { + for _ in 0..COUNT { + let c = self.current_char.ok_or(ScannerError::HexadecimalNotFound)?; + + if !is_hex_digit(c) { + return Err(ScannerError::HexadecimalNotFound); + } + + self.next(); + } + + Ok(()) + } +} + +impl<'a> IntoIterator for Scanner<'a> { + type Item = Result>; + type IntoIter = IntoIter<'a>; + + fn into_iter(self) -> Self::IntoIter { + Self::IntoIter::new(self) + } +} + +pub struct IntoIter<'a> { + scanner: Scanner<'a>, + done: bool, +} + +impl<'a> IntoIter<'a> { + const fn new(scanner: Scanner<'a>) -> Self { + Self { scanner, done: false } + } +} + +impl<'a> Iterator for IntoIter<'a> { + type Item = Result>; + + fn next(&mut self) -> Option { + if self.done { + return None; + } + + match self.scanner.scan() { + Ok((pos, tok, lit)) => { + if tok == Token::EOF { + self.done = true; + } + Some(Ok((pos, tok, lit))) + } + Err(err) => { + self.done = true; + Some(Err(err)) + } + } + } +} + +// https://golang.org/ref/spec#Letters_and_digits + +fn is_letter(c: char) -> bool { + c == '_' || is_unicode_letter(c) +} + +//const fn is_decimal_digit(c: char) -> bool { +//matches!(c, '0'..='9') +//} + +//const fn is_binary_digit(c: char) -> bool { +//matches!(c, '0'..='1') +//} + +const fn is_octal_digit(c: char) -> bool { + matches!(c, '0'..='7') +} + +const fn is_hex_digit(c: char) -> bool { + matches!(c, '0'..='9' | 'A'..='F' | 'a'..='f') +} + +// https://golang.org/ref/spec#Characters + +const fn is_newline(c: char) -> bool { + c == '\n' +} + +//const fn is_unicode_char(c: char) -> bool { +//c != '\n' +//} + +fn is_unicode_letter(c: char) -> bool { + matches!( + get_general_category(c), + GeneralCategory::UppercaseLetter + | GeneralCategory::LowercaseLetter + | GeneralCategory::TitlecaseLetter + | GeneralCategory::ModifierLetter + | GeneralCategory::OtherLetter + ) +} + +fn is_unicode_digit(c: char) -> bool { + get_general_category(c) == GeneralCategory::DecimalNumber +} + +// https://golang.org/ref/spec#Keywords + +static KEYWORDS: Map<&'static str, Token> = phf_map! { + "break" => Token::BREAK, + "case" => Token::CASE, + "chan" => Token::CHAN, + "const" => Token::CONST, + "continue" => Token::CONTINUE, + + "default" => Token::DEFAULT, + "defer" => Token::DEFER, + "else" => Token::ELSE, + "fallthrough" => Token::FALLTHROUGH, + "for" => Token::FOR, + + "func" => Token::FUNC, + "go" => Token::GO, + "goto" => Token::GOTO, + "if" => Token::IF, + "import" => Token::IMPORT, + + "interface" => Token::INTERFACE, + "map" => Token::MAP, + "package" => Token::PACKAGE, + "range" => Token::RANGE, + "return" => Token::RETURN, + + "select" => Token::SELECT, + "struct" => Token::STRUCT, + "switch" => Token::SWITCH, + "type" => Token::TYPE, + "var" => Token::VAR, +}; + +#[cfg(test)] +mod tests { + use super::Scanner; + + #[test] // fuzz + fn it_should_return_an_error_on_missing_line_number() { + let input = "/*line :*/"; + let mut out: Vec<_> = Scanner::new(file!(), input).into_iter().collect(); + assert!(out.pop().unwrap().is_err()); + } +} diff --git a/backend/src/parser_go_token.rs b/backend/src/parser_go_token.rs new file mode 100644 index 0000000000..5d63605374 --- /dev/null +++ b/backend/src/parser_go_token.rs @@ -0,0 +1,273 @@ +// https://cs.opensource.google/go/go/+/refs/tags/go1.17.2:src/go/token/token.go + +#![allow(non_camel_case_types)] // For consistency with the Go tokens + +use std::fmt; + +#[derive(Clone, Copy, Debug, Default)] +pub struct Position<'a> { + pub directory: &'a str, + pub file: &'a str, + pub offset: usize, + pub line: usize, + pub column: usize, +} + +impl<'a> fmt::Display for Position<'a> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.file.is_empty() { + write!(f, ":{}:{}", self.line, self.column) + } else if self.file.starts_with('/') { + write!(f, "{}:{}:{}", self.file, self.line, self.column) + } else { + write!( + f, + "{}/{}:{}:{}", + self.directory, self.file, self.line, self.column + ) + } + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum Token { + EOF, + COMMENT, + + IDENT, // main + INT, // 12345 + FLOAT, // 123.45 + IMAG, // 123.45i + CHAR, // 'a' + STRING, // "abc" + + ADD, // + + SUB, // - + MUL, // * + QUO, // / + REM, // % + + AND, // & + OR, // | + XOR, // ^ + SHL, // << + SHR, // >> + AND_NOT, // &^ + + ADD_ASSIGN, // += + SUB_ASSIGN, // -= + MUL_ASSIGN, // *= + QUO_ASSIGN, // /= + REM_ASSIGN, // %= + + AND_ASSIGN, // &= + OR_ASSIGN, // |= + XOR_ASSIGN, // ^= + SHL_ASSIGN, // <<= + SHR_ASSIGN, // >>= + AND_NOT_ASSIGN, // &^= + + LAND, // && + LOR, // || + ARROW, // <- + INC, // ++ + DEC, // -- + + EQL, // == + LSS, // < + GTR, // > + ASSIGN, // = + NOT, // ! + + NEQ, // != + LEQ, // <= + GEQ, // >= + DEFINE, // := + ELLIPSIS, // ... + + LPAREN, // ( + LBRACK, // [ + LBRACE, // { + COMMA, // , + PERIOD, // . + + RPAREN, // ) + RBRACK, // ] + RBRACE, // } + SEMICOLON, // ; + COLON, // : + + BREAK, + CASE, + CHAN, + CONST, + CONTINUE, + + DEFAULT, + DEFER, + ELSE, + FALLTHROUGH, + FOR, + + FUNC, + GO, + GOTO, + IF, + IMPORT, + + INTERFACE, + MAP, + PACKAGE, + RANGE, + RETURN, + + SELECT, + STRUCT, + SWITCH, + TYPE, + VAR, +} + +impl Token { + pub const fn is_assign_op(&self) -> bool { + use Token::*; + matches!( + self, + ADD_ASSIGN + | SUB_ASSIGN + | MUL_ASSIGN + | QUO_ASSIGN + | REM_ASSIGN + | AND_ASSIGN + | OR_ASSIGN + | XOR_ASSIGN + | SHL_ASSIGN + | SHR_ASSIGN + | AND_NOT_ASSIGN + ) + } + + // https://go.dev/ref/spec#Operator_precedence + pub fn precedence(&self) -> u8 { + use Token::*; + match self { + MUL | QUO | REM | SHL | SHR | AND | AND_NOT => 5, + ADD | SUB | OR | XOR => 4, + EQL | NEQ | LSS | LEQ | GTR | GEQ => 3, + LAND => 2, + LOR => 1, + _ => unreachable!( + "precedence() is only supported for binary operators, called with: {:?}", + self + ), + } + } + + pub const fn lowest_precedence() -> u8 { + 0 + } +} + +impl From<&Token> for &'static str { + fn from(token: &Token) -> Self { + use Token::*; + + match token { + EOF => "EOF", + COMMENT => "COMMENT", + + IDENT => "IDENT", + INT => "INT", + FLOAT => "FLOAT", + IMAG => "IMAG", + CHAR => "CHAR", + STRING => "STRING", + + ADD => "+", + SUB => "-", + MUL => "*", + QUO => "/", + REM => "%", + + AND => "&", + OR => "|", + XOR => "^", + SHL => "<<", + SHR => ">>", + AND_NOT => "&^", + + ADD_ASSIGN => "+=", + SUB_ASSIGN => "-=", + MUL_ASSIGN => "*=", + QUO_ASSIGN => "/=", + REM_ASSIGN => "%=", + + AND_ASSIGN => "&=", + OR_ASSIGN => "|=", + XOR_ASSIGN => "^=", + SHL_ASSIGN => "<<=", + SHR_ASSIGN => ">>=", + AND_NOT_ASSIGN => "&^=", + + LAND => "&&", + LOR => "||", + ARROW => "<-", + INC => "++", + DEC => "--", + + EQL => "==", + LSS => "<", + GTR => ">", + ASSIGN => "=", + NOT => "!", + + NEQ => "!=", + LEQ => "<=", + GEQ => ">=", + DEFINE => ":=", + ELLIPSIS => "...", + + LPAREN => "(", + LBRACK => "[", + LBRACE => "{", + COMMA => ",", + PERIOD => ".", + + RPAREN => ")", + RBRACK => "]", + RBRACE => "}", + SEMICOLON => ";", + COLON => ":", + + BREAK => "break", + CASE => "case", + CHAN => "chan", + CONST => "const", + CONTINUE => "continue", + + DEFAULT => "default", + DEFER => "defer", + ELSE => "else", + FALLTHROUGH => "fallthrough", + FOR => "for", + + FUNC => "func", + GO => "go", + GOTO => "goto", + IF => "if", + IMPORT => "import", + + INTERFACE => "interface", + MAP => "map", + PACKAGE => "package", + RANGE => "range", + RETURN => "return", + + SELECT => "select", + STRUCT => "struct", + SWITCH => "switch", + TYPE => "type", + VAR => "var", + } + } +} diff --git a/backend/src/parser_py.rs b/backend/src/parser_py.rs index 4ff8f7beba..72f502fe82 100644 --- a/backend/src/parser_py.rs +++ b/backend/src/parser_py.rs @@ -101,6 +101,7 @@ pub fn parse_python_signature(code: &str) -> error::Result { None }; Arg { + otyp: None, name: x.arg, typ: x.annotation.map_or(Typ::Unknown, |e| match *e { Located { location: _, node: ExpressionType::Identifier { name } } => { @@ -269,18 +270,21 @@ def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = byt star_kwargs: false, args: vec![ Arg { + otyp: None, name: "test1".to_string(), typ: Typ::Str(None), default: None, has_default: false }, Arg { + otyp: None, name: "name".to_string(), typ: Typ::Unknown, default: Some(json!("")), has_default: true }, Arg { + otyp: None, name: "byte".to_string(), typ: Typ::Bytes, default: Some(json!("")), @@ -316,18 +320,21 @@ def main(test1: str, star_kwargs: false, args: vec![ Arg { + otyp: None, name: "test1".to_string(), typ: Typ::Str(None), default: None, has_default: false }, Arg { + otyp: None, name: "name".to_string(), typ: Typ::Unknown, default: Some(json!("")), has_default: true }, Arg { + otyp: None, name: "byte".to_string(), typ: Typ::Bytes, default: Some(json!("")), @@ -359,18 +366,21 @@ def main(test1: str, star_kwargs: false, args: vec![ Arg { + otyp: None, name: "test1".to_string(), typ: Typ::Str(None), default: None, has_default: false }, Arg { + otyp: None, name: "name".to_string(), typ: Typ::Unknown, default: Some(json!("")), has_default: true }, Arg { + otyp: None, name: "byte".to_string(), typ: Typ::Bytes, default: Some(json!("")), diff --git a/backend/src/parser_ts.rs b/backend/src/parser_ts.rs index d767834d09..80c62829b3 100644 --- a/backend/src/parser_ts.rs +++ b/backend/src/parser_ts.rs @@ -68,6 +68,7 @@ pub fn parse_deno_signature(code: &str) -> error::Result { Pat::Ident(ident) => { let (name, typ, nullable) = binding_ident_to_arg(&ident); Ok(Arg { + otyp: None, name, typ, default: None, @@ -106,7 +107,7 @@ pub fn parse_deno_signature(code: &str) -> error::Result { if typ == Typ::Unknown && default.is_some() { typ = json_to_typ(default.as_ref().unwrap()); } - Ok(Arg { name, typ, default, has_default: true }) + Ok(Arg { otyp: None, name, typ, default, has_default: true }) } _ => Err(error::Error::ExecutionErr(format!( "parameter syntax unsupported: `{}`", @@ -288,72 +289,84 @@ export function main(test1?: string, test2: string = \"burkina\", star_kwargs: false, args: vec![ Arg { + otyp: None, name: "test1".to_string(), typ: Typ::Str(None), default: None, has_default: true }, Arg { + otyp: None, name: "test2".to_string(), typ: Typ::Str(None), default: Some(json!("burkina")), has_default: true }, Arg { + otyp: None, name: "test3".to_string(), typ: Typ::Resource("postgres".to_string()), default: None, has_default: false }, Arg { + otyp: None, name: "b64".to_string(), typ: Typ::Bytes, default: None, has_default: false }, Arg { + otyp: None, name: "ls".to_string(), typ: Typ::List(Box::new(Typ::Bytes)), default: None, has_default: false }, Arg { + otyp: None, name: "email".to_string(), typ: Typ::Email, default: None, has_default: false }, Arg { + otyp: None, name: "literal".to_string(), typ: Typ::Str(Some(vec!["test".to_string()])), default: None, has_default: false }, Arg { + otyp: None, name: "literal_union".to_string(), typ: Typ::Str(Some(vec!["test".to_string(), "test2".to_string()])), default: None, has_default: false }, Arg { + otyp: None, name: "opt_type".to_string(), typ: Typ::Str(None), default: None, has_default: true }, Arg { + otyp: None, name: "opt_type_union".to_string(), typ: Typ::Str(None), default: None, has_default: true }, Arg { + otyp: None, name: "opt_type_union_union2".to_string(), typ: Typ::Str(None), default: None, has_default: true }, Arg { + otyp: None, name: "min_object".to_string(), typ: Typ::Object(vec![ ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) }, @@ -388,36 +401,42 @@ export function main(test2 = \"burkina\", star_kwargs: false, args: vec![ Arg { + otyp: None, name: "test2".to_string(), typ: Typ::Str(None), default: Some(json!("burkina")), has_default: true }, Arg { + otyp: None, name: "bool".to_string(), typ: Typ::Bool, default: Some(json!(true)), has_default: true }, Arg { + otyp: None, name: "float".to_string(), typ: Typ::Float, default: Some(json!(4.2)), has_default: true }, Arg { + otyp: None, name: "int".to_string(), typ: Typ::Int, default: Some(json!(42)), has_default: true }, Arg { + otyp: None, name: "ls".to_string(), typ: Typ::List(Box::new(Typ::Str(None))), default: Some(json!(["test"])), has_default: true }, Arg { + otyp: None, name: "min_object".to_string(), typ: Typ::Object(vec![ ObjectProperty { key: "a".to_string(), typ: Box::new(Typ::Str(None)) }, diff --git a/backend/src/scripts.rs b/backend/src/scripts.rs index 9502966498..73844e8a3e 100644 --- a/backend/src/scripts.rs +++ b/backend/src/scripts.rs @@ -14,7 +14,7 @@ use crate::{ audit::{audit_log, ActionKind}, db::{UserDB, DB}, error::{to_anyhow, Error, JsonResult, Result}, - jobs, parser, parser_py, parser_ts, + jobs, parser, parser_go, parser_py, parser_ts, users::{owner_to_token_owner, truncate_token, Authed, Tokened}, utils::{http_get_from_hub, list_elems_from_hub, require_admin, Pagination, StripPath}, }; @@ -43,6 +43,7 @@ pub fn global_service() -> Router { post(parse_python_code_to_jsonschema), ) .route("/deno/tojsonschema", post(parse_deno_code_to_jsonschema)) + .route("/go/tojsonschema", post(parse_go_code_to_jsonschema)) .route("/hub/list", get(list_hub_scripts)) .route("/hub/get/*path", get(get_hub_script_by_path)) } @@ -68,6 +69,7 @@ pub fn workspaced_service() -> Router { pub enum ScriptLang { Deno, Python3, + Go, } impl ScriptLang { @@ -75,6 +77,7 @@ impl ScriptLang { match self { ScriptLang::Deno => "deno", ScriptLang::Python3 => "python3", + ScriptLang::Go => "go", } } } @@ -436,12 +439,15 @@ async fn create_script( .execute(&mut tx) .await?; - let mut tx = if ns.lock.is_none() && ns.language == ScriptLang::Python3 { - let dependencies = parser_py::parse_python_imports(&ns.content)?; + let mut tx = if ns.lock.is_none() && ns.language != ScriptLang::Deno { + let dependencies = match ns.language { + ScriptLang::Python3 => parser_py::parse_python_imports(&ns.content)?.join("\n"), + _ => ns.content, + }; let (_, tx) = jobs::push( tx, &w_id, - jobs::JobPayload::Dependencies { hash, dependencies }, + jobs::JobPayload::Dependencies { hash, dependencies, language: ns.language }, None, &authed.username, owner_to_token_owner(&authed.username, false), @@ -772,6 +778,11 @@ async fn parse_deno_code_to_jsonschema( ) -> JsonResult { parser_ts::parse_deno_signature(&code).map(Json) } +async fn parse_go_code_to_jsonschema( + Json(code): Json, +) -> JsonResult { + parser_go::parse_go_sig(&code).map(Json) +} pub fn to_i64(s: &str) -> Result { let v = hex::decode(s)?; diff --git a/backend/src/worker.rs b/backend/src/worker.rs index 27b70d85a5..5d4e03d864 100644 --- a/backend/src/worker.rs +++ b/backend/src/worker.rs @@ -16,6 +16,7 @@ use std::{ }, time::Duration, }; +use uuid::Uuid; use crate::{ db::DB, @@ -49,14 +50,20 @@ use async_recursion::async_recursion; const TMP_DIR: &str = "/tmp/windmill"; const PIP_CACHE_DIR: &str = "/tmp/windmill/cache/pip"; const DENO_CACHE_DIR: &str = "/tmp/windmill/cache/deno"; +const GO_CACHE_DIR: &str = "/tmp/windmill/cache/go"; const NUM_SECS_ENV_CHECK: u64 = 15; -const INCLUDE_DEPS_SH_CONTENT: &str = include_str!("../../nsjail/download_deps.sh"); -const NSJAIL_CONFIG_DOWNLOAD_CONTENT: &str = include_str!("../../nsjail/download.config.proto"); +const INCLUDE_DEPS_PY_SH_CONTENT: &str = include_str!("../../nsjail/download_deps.py.sh"); +const NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT: &str = + include_str!("../../nsjail/download.py.config.proto"); const NSJAIL_CONFIG_RUN_PYTHON3_CONTENT: &str = include_str!("../../nsjail/run.python3.config.proto"); + +const NSJAIL_CONFIG_RUN_GO_CONTENT: &str = include_str!("../../nsjail/run.go.config.proto"); + const NSJAIL_CONFIG_RUN_DENO_CONTENT: &str = include_str!("../../nsjail/run.deno.config.proto"); const MAX_LOG_SIZE: u32 = 200000; +const GO_REQ_SPLITTER: &str = "//go.sum"; pub struct Metrics { pub jobs_failed: prometheus::IntCounter, } @@ -67,6 +74,7 @@ pub struct WorkerConfig { pub base_url: String, pub disable_nuser: bool, pub disable_nsjail: bool, + pub keep_job_dir: bool, } pub async fn run_worker( @@ -84,7 +92,7 @@ pub async fn run_worker( let worker_dir = format!("{TMP_DIR}/{worker_name}"); tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir"); - for x in [&worker_dir, PIP_CACHE_DIR, DENO_CACHE_DIR] { + for x in [&worker_dir, PIP_CACHE_DIR, DENO_CACHE_DIR, GO_CACHE_DIR] { DirBuilder::new() .recursive(true) .create(x) @@ -92,7 +100,12 @@ pub async fn run_worker( .expect("could not create initial worker dir"); } - let _ = write_file(&worker_dir, "download_deps.sh", INCLUDE_DEPS_SH_CONTENT).await; + let _ = write_file( + &worker_dir, + "download_deps.py.sh", + INCLUDE_DEPS_PY_SH_CONTENT, + ) + .await; let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_ENV_CHECK + 1); @@ -132,11 +145,15 @@ pub async fn run_worker( let mut jobs_executed = 0; let deno_path = std::env::var("DENO_PATH").unwrap_or_else(|_| "/usr/bin/deno".to_string()); + let go_path = std::env::var("GO_PATH").unwrap_or_else(|_| "/usr/bin/go".to_string()); let python_path = std::env::var("PYTHON_PATH").unwrap_or_else(|_| "/usr/local/bin/python3".to_string()); let nsjail_path = std::env::var("NSJAIL_PATH").unwrap_or_else(|_| "nsjail".to_string()); let path_env = std::env::var("PATH").unwrap_or_else(|_| String::new()); - let envs = Envs { deno_path, python_path, nsjail_path, path_env }; + let gopath_env = std::env::var("GOPATH").unwrap_or_else(|_| String::new()); + let home_env = std::env::var("HOME").unwrap_or_else(|_| String::new()); + let envs = + Envs { deno_path, go_path, python_path, nsjail_path, path_env, gopath_env, home_env }; loop { if last_ping.elapsed().as_secs() > NUM_SECS_ENV_CHECK { @@ -176,7 +193,7 @@ pub async fn run_worker( timeout, &worker_name, &worker_dir, - worker_config.clone(), + &worker_config, &metrics, &envs, ) @@ -274,9 +291,12 @@ async fn insert_initial_ping(worker_instance: &str, worker_name: &str, ip: &str, struct Envs { deno_path: String, + go_path: String, python_path: String, nsjail_path: String, path_env: String, + gopath_env: String, + home_env: String, } async fn handle_queued_job( job: QueuedJob, @@ -284,7 +304,7 @@ async fn handle_queued_job( timeout: i32, worker_name: &str, worker_dir: &str, - worker_config: WorkerConfig, + worker_config: &WorkerConfig, metrics: &Metrics, envs: &Envs, ) -> crate::error::Result<()> { @@ -417,8 +437,8 @@ async fn handle_job( worker_dir: &str, logs: &mut String, last_line: &mut String, - worker_config: WorkerConfig, - Envs { deno_path, python_path, nsjail_path, path_env }: &Envs, + worker_config: &WorkerConfig, + envs: &Envs, ) -> Result { tracing::info!( worker = %worker_name, @@ -430,36 +450,46 @@ async fn handle_job( logs.push_str(&format!("job {} on worker {}\n", &job.id, &worker_name)); let job_dir = format!("{worker_dir}/{}", job.id); + DirBuilder::new() - .recursive(true) - .create(&format!("{job_dir}/dependencies")) + .create(&job_dir) .await - .expect("could not create initial job dir"); + .expect("could not create job dir"); let mut status: Result = Err(Error::InternalErr("job not started".to_string())); if matches!(job.job_kind, JobKind::Dependencies) { - handle_dependency_job(job, logs, &job_dir, &mut status, db, last_line, timeout).await?; + handle_dependency_job( + job, + logs, + &job_dir, + &mut status, + db, + last_line, + timeout, + &envs.go_path, + ) + .await?; } else { - handle_nondep_job( + handle_code_execution_job( job, db, &job_dir, worker_dir, - worker_config, logs, &mut status, last_line, timeout, - deno_path, - python_path, - nsjail_path, - path_env, + worker_config, + envs, ) .await?; } - tokio::fs::remove_dir_all(job_dir).await?; + + if !worker_config.keep_job_dir { + tokio::fs::remove_dir_all(job_dir).await?; + } if status.is_ok() && status.as_ref().unwrap().success() { let result = serde_json::from_str::(last_line).map_err(|e| { @@ -488,34 +518,25 @@ async fn handle_job( } } -async fn handle_nondep_job( +async fn handle_code_execution_job( job: &QueuedJob, db: &sqlx::Pool, job_dir: &String, worker_dir: &str, - WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail }: WorkerConfig, logs: &mut String, status: &mut Result, last_line: &mut String, timeout: i32, - deno_path: &str, - python_path: &str, - nsjail_path: &str, - path_env: &str, + worker_config: &WorkerConfig, + envs: &Envs, ) -> Result<(), Error> { let (inner_content, requirements_o, language) = if matches!(job.job_kind, JobKind::Preview) || matches!(job.job_kind, JobKind::Script_Hub) { let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned(); - let reqs = if job - .language - .as_ref() - .map(|x| matches!(x, ScriptLang::Python3)) - .unwrap_or(false) - { - Some(parser_py::parse_python_imports(&code)?.join("\n")) - } else { - None + let reqs = match job.language { + Some(ScriptLang::Python3) => Some(parser_py::parse_python_imports(&code)?.join("\n")), + _ => None, }; (code, reqs, job.language.to_owned()) } else { @@ -530,6 +551,21 @@ async fn handle_nondep_job( .ok_or_else(|| Error::InternalErr(format!("expected content and lock")))? }; let worker_name = worker_dir.split("/").last().unwrap_or("unknown"); + let lang_str = job + .language + .as_ref() + .map(|x| format!("{x:?}")) + .unwrap_or_else(|| "NO_LANG".to_string()); + + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + is_ok = status.is_ok(), + "started {} job {}", + &lang_str, + job.id + ); match language { None => { return Err(Error::ExecutionErr( @@ -537,131 +573,492 @@ async fn handle_nondep_job( ))?; } Some(ScriptLang::Python3) => { - let requirements = - requirements_o.ok_or_else(|| Error::InternalErr(format!("lockfile missing")))?; + handle_python_job( + worker_config, + envs, + requirements_o, + job_dir, + worker_dir, + worker_name, + job, + logs, + status, + db, + last_line, + timeout, + &inner_content, + ) + .await? + } + Some(ScriptLang::Deno) => { + handle_deno_job( + worker_config, + envs, + logs, + job, + db, + job_dir, + &inner_content, + timeout, + status, + last_line, + ) + .await?; + } + Some(ScriptLang::Go) => { + handle_go_job( + worker_config, + envs, + logs, + job, + db, + &inner_content, + timeout, + job_dir, + requirements_o, + status, + last_line, + ) + .await? + } + } + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + is_ok = status.is_ok(), + "finished {} job {}", + &lang_str, + job.id + ); + Ok(()) +} - if requirements.len() > 0 { - if !disable_nsjail { - let _ = write_file( - job_dir, - "download.config.proto", - &NSJAIL_CONFIG_DOWNLOAD_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{WORKER_DIR}", &worker_dir) - .replace("{CACHE_DIR}", PIP_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), - ) - .await?; - } - let _ = write_file(job_dir, "requirements.txt", &requirements).await?; +async fn handle_go_job( + WorkerConfig { base_internal_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig, + Envs { nsjail_path, go_path, path_env, gopath_env, home_env, .. }: &Envs, + logs: &mut String, + job: &QueuedJob, + db: &sqlx::Pool, + inner_content: &str, + timeout: i32, + job_dir: &String, + requirements_o: Option, + status: &mut Result, + last_line: &mut String, +) -> Result<(), Error> { + //go does not like executing modules at temp root + let job_dir = &format!("{job_dir}/go"); + if let Some(requirements) = requirements_o { + gen_go_mymod(inner_content, job_dir).await?; + let (md, sum) = requirements + .split_once(GO_REQ_SPLITTER) + .ok_or(Error::ExecutionErr( + "Invalid requirement file, missing splitter".to_string(), + ))?; + write_file(job_dir, "go.mod", md).await?; + write_file(job_dir, "go.sum", sum).await?; + } else { + logs.push_str("\n\n--- GO DEPENDENCIES SETUP---\n"); + set_logs(logs, job.id, db).await; - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "started setup python dependencies" - ); + install_go_dependencies( + &job.id, + inner_content, + logs, + job_dir, + status, + db, + last_line, + timeout, + go_path, + true, + ) + .await?; + } - let child = if !disable_nsjail { - Command::new(nsjail_path) - .current_dir(job_dir) - .args(vec!["--config", "download.config.proto"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - Command::new(python_path) - .current_dir(job_dir) - .args(vec![ - "-m", - "pip", - "install", - "--no-color", - "--isolated", - "--no-warn-conflicts", - "--disable-pip-version-check", - "-t", - "./dependencies", - "-r", - "./requirements.txt", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; + logs.push_str("\n\n--- GO CODE EXECUTION ---\n"); + set_logs(logs, job.id, db).await; - logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n"); - *status = handle_child(job, db, logs, last_line, timeout, child).await; - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - is_ok = status.is_ok(), - "finished setting up python dependencies {}", - job.id - ); - } - if requirements.len() == 0 || status.is_ok() { - logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n"); + let sig = crate::parser_go::parse_go_sig(&inner_content)?; + let token = create_token_for_owner( + &db, + &job.workspace_id, + &job.permissioned_as, + "ephemeral-script", + timeout * 2, + &job.created_by, + ) + .await?; + let args = if let Some(args) = &job.args { + Some( + transform_json_value(&token, &job.workspace_id, &base_internal_url, args.clone()) + .await?, + ) + } else { + None + }; + let ser_args = serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; + write_file(job_dir, "args.json", &ser_args).await?; + let spread = sig + .args + .into_iter() + .map(|x| { + format!( + "json_arg[\"{}\"].({})", + x.name, + x.otyp.unwrap_or_else(|| "interface{}".to_string()) + ) + }) + .join(", "); - set_logs(logs, job.id, db).await; + let inner_content = inner_content.replace("func main(", "func inner_main("); + let wrapper_content: String = format!( + r#" +package main - let _ = write_file(job_dir, "inner.py", &inner_content).await?; +import ( + "encoding/json" + "os" +) - let sig = crate::parser_py::parse_python_signature(&inner_content)?; - let transforms = sig - .args - .into_iter() - .map(|x| match x.typ { - Typ::Bytes => { - format!( - "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ +{inner_content} + +func main() {{ + dat, err := os.ReadFile("args.json") + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + + var json_arg map[string]interface{{}} + + if err := json.Unmarshal(dat, &json_arg); err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + + res, err := inner_main({spread}) + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + res_json, err := json.Marshal(res) + if err != nil {{ + fmt.Println(err) + os.Exit(1) + }} + fmt.Println() + fmt.Println("result:") + fmt.Println(string(res_json)) +}} + +"#, + ); + write_file(job_dir, "mymod/main.go", &wrapper_content).await?; + let mut reserved_variables = get_reserved_variables(job, token.clone(), db).await?; + reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); + + let child = if !disable_nsjail { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_GO_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", GO_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), + ) + .await?; + + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec![ + "--config", + "run.config.proto", + "--", + go_path, + "run", + "mymod/main.go", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(go_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .env("GOPATH", gopath_env) + .env("HOME", home_env) + .args(vec!["run", "mymod/main.go"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + *status = handle_child(&job.id, db, logs, last_line, timeout, child).await; + Ok(()) +} + +async fn handle_deno_job( + WorkerConfig { base_internal_url, base_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig, + Envs { nsjail_path, deno_path, path_env, .. }: &Envs, + logs: &mut String, + job: &QueuedJob, + db: &sqlx::Pool, + job_dir: &String, + inner_content: &String, + timeout: i32, + status: &mut Result, + last_line: &mut String, +) -> Result<(), Error> { + logs.push_str("\n\n--- DENO CODE EXECUTION ---\n"); + set_logs(logs, job.id, db).await; + let _ = write_file(job_dir, "inner.ts", inner_content).await?; + let sig = crate::parser_ts::parse_deno_signature(inner_content)?; + let token = create_token_for_owner( + &db, + &job.workspace_id, + &job.permissioned_as, + "ephemeral-script", + timeout * 2, + &job.created_by, + ) + .await?; + let args = if let Some(args) = &job.args { + Some( + transform_json_value(&token, &job.workspace_id, &base_internal_url, args.clone()) + .await?, + ) + } else { + None + }; + let ser_args = serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; + write_file(job_dir, "args.json", &ser_args).await?; + let spread = sig.args.into_iter().map(|x| x.name).join(","); + let wrapper_content: String = format!( + r#" +import {{ main }} from "./inner.ts"; + +const args = await Deno.readTextFile("args.json") + .then(JSON.parse) + .then(({{ {spread} }}) => [ {spread} ]) + +async function run() {{ + let res: any = await main(...args); + const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); + console.log(); + console.log("result:"); + console.log(res_json); + Deno.exit(0); +}} +run(); +"#, + ); + write_file(job_dir, "main.ts", &wrapper_content).await?; + let mut reserved_variables = get_reserved_variables(job, token.clone(), db).await?; + reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); + + let hostname_base = base_url.split("://").last().unwrap_or("localhost"); + let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost"); + let deno_auth_tokens = format!("{token}@{hostname_base};{token}@{hostname_internal}"); + let child = if !disable_nsjail { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_DENO_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CACHE_DIR}", DENO_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), + ) + .await?; + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("DENO_AUTH_TOKENS", deno_auth_tokens) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec![ + "--config", + "run.config.proto", + "--", + deno_path, + "run", + "--unstable", + "--v8-flags=--max-heap-size=2048", + "-A", + "/tmp/main.ts", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(deno_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("DENO_AUTH_TOKENS", deno_auth_tokens) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec![ + "run", + "--unstable", + "--v8-flags=--max-heap-size=2048", + "-A", + "main.ts", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + *status = handle_child(&job.id, db, logs, last_line, timeout, child).await; + + Ok(()) +} + +async fn handle_python_job( + WorkerConfig { base_internal_url, disable_nuser, disable_nsjail, .. }: &WorkerConfig, + Envs { nsjail_path, python_path, path_env, .. }: &Envs, + requirements_o: Option, + job_dir: &String, + worker_dir: &str, + worker_name: &str, + job: &QueuedJob, + logs: &mut String, + status: &mut Result, + db: &sqlx::Pool, + last_line: &mut String, + timeout: i32, + inner_content: &String, +) -> Result<(), Error> { + let requirements = + requirements_o.ok_or_else(|| Error::InternalErr(format!("lockfile missing")))?; + + create_dependencies_dir(job_dir).await; + + if requirements.len() > 0 { + if !disable_nsjail { + let _ = write_file( + job_dir, + "download.config.proto", + &NSJAIL_CONFIG_DOWNLOAD_PY_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{WORKER_DIR}", &worker_dir) + .replace("{CACHE_DIR}", PIP_CACHE_DIR) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), + ) + .await?; + } + let _ = write_file(job_dir, "requirements.txt", &requirements).await?; + + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + "started setup python dependencies" + ); + + let child = if !disable_nsjail { + Command::new(nsjail_path) + .current_dir(job_dir) + .args(vec!["--config", "download.config.proto"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(python_path) + .current_dir(job_dir) + .args(vec![ + "-m", + "pip", + "install", + "--no-color", + "--isolated", + "--no-warn-conflicts", + "--disable-pip-version-check", + "-t", + "./dependencies", + "-r", + "./requirements.txt", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + + logs.push_str("\n--- PIP DEPENDENCIES INSTALL ---\n"); + *status = handle_child(&job.id, db, logs, last_line, timeout, child).await; + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + is_ok = status.is_ok(), + "finished setting up python dependencies {}", + job.id + ); + } + if requirements.len() == 0 || status.is_ok() { + logs.push_str("\n\n--- PYTHON CODE EXECUTION ---\n"); + + set_logs(logs, job.id, db).await; + + let _ = write_file(job_dir, "inner.py", inner_content).await?; + + let sig = crate::parser_py::parse_python_signature(inner_content)?; + let transforms = sig + .args + .into_iter() + .map(|x| match x.typ { + Typ::Bytes => { + format!( + "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", - x.name, x.name, x.name, x.name - ) - } - Typ::Datetime => { - format!( - "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ + x.name, x.name, x.name, x.name + ) + } + Typ::Datetime => { + format!( + "if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n \ kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], \ '%Y-%m-%dT%H:%M')\n", - x.name, x.name, x.name, x.name - ) - } - _ => "".to_string(), - }) - .collect::>() - .join(""); - - let token = create_token_for_owner( - &db, - &job.workspace_id, - &job.permissioned_as, - "ephemeral-script", - timeout * 2, - &job.created_by, - ) - .await?; - - let args = if let Some(args) = &job.args { - Some( - transform_json_value( - &token, - &job.workspace_id, - &base_internal_url, - args.clone(), - ) - .await?, + x.name, x.name, x.name, x.name ) - } else { - None - }; - let ser_args = - serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; - write_file(job_dir, "args.json", &ser_args).await?; + } + _ => "".to_string(), + }) + .collect::>() + .join(""); - let wrapper_content: String = format!( - r#" + let token = create_token_for_owner( + &db, + &job.workspace_id, + &job.permissioned_as, + "ephemeral-script", + timeout * 2, + &job.created_by, + ) + .await?; + + let args = if let Some(args) = &job.args { + Some( + transform_json_value(&token, &job.workspace_id, &base_internal_url, args.clone()) + .await?, + ) + } else { + None + }; + let ser_args = + serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; + write_file(job_dir, "args.json", &ser_args).await?; + + let wrapper_content: String = format!( + r#" import json import base64 from datetime import datetime @@ -680,270 +1077,238 @@ print() print("result:") print(res_json) "#, - ); - write_file(job_dir, "main.py", &wrapper_content).await?; + ); + write_file(job_dir, "main.py", &wrapper_content).await?; - let mut reserved_variables = get_reserved_variables(job, token, db).await?; - if !disable_nsjail { - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), - ) - .await?; - } else { - reserved_variables - .insert("PYTHONPATH".to_string(), format!("{job_dir}/dependencies")); - } - - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "started python code execution {}", - job.id - ); - let child = if !disable_nsjail { - Command::new(nsjail_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec![ - "--config", - "run.config.proto", - "--", - python_path, - "-u", - "/tmp/main.py", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - Command::new(python_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec!["-u", "main.py"]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; - - *status = handle_child(job, db, logs, last_line, timeout, child).await; - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - is_ok = status.is_ok(), - "finished python code execution {}", - job.id - ); - } - } - Some(ScriptLang::Deno) => { - logs.push_str("\n\n--- DENO CODE EXECUTION ---\n"); - - set_logs(logs, job.id, db).await; - - let _ = write_file(job_dir, "inner.ts", &inner_content).await?; - - let sig = crate::parser_ts::parse_deno_signature(&inner_content)?; - - let token = create_token_for_owner( - &db, - &job.workspace_id, - &job.permissioned_as, - "ephemeral-script", - timeout * 2, - &job.created_by, + let mut reserved_variables = get_reserved_variables(job, token, db).await?; + if !disable_nsjail { + let _ = write_file( + job_dir, + "run.config.proto", + &NSJAIL_CONFIG_RUN_PYTHON3_CONTENT + .replace("{JOB_DIR}", job_dir) + .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), ) .await?; - - let args = if let Some(args) = &job.args { - Some( - transform_json_value( - &token, - &job.workspace_id, - &base_internal_url, - args.clone(), - ) - .await?, - ) - } else { - None - }; - let ser_args = - serde_json::to_string(&args).map_err(|e| Error::ExecutionErr(e.to_string()))?; - write_file(job_dir, "args.json", &ser_args).await?; - - let spread = sig.args.into_iter().map(|x| x.name).join(","); - let wrapper_content: String = format!( - r#" -import {{ main }} from "./inner.ts"; - -const args = await Deno.readTextFile("args.json") - .then(JSON.parse) - .then(({{ {spread} }}) => [ {spread} ]) - -async function run() {{ - let res: any = await main(...args); - const res_json = JSON.stringify(res ?? null, (key, value) => typeof value === 'undefined' ? null : value); - console.log(); - console.log("result:"); - console.log(res_json); - Deno.exit(0); -}} -run(); -"#, - ); - write_file(job_dir, "main.ts", &wrapper_content).await?; - - let mut reserved_variables = get_reserved_variables(job, token.clone(), db).await?; - reserved_variables.insert("RUST_LOG".to_string(), "info".to_string()); - - if !disable_nsjail { - let _ = write_file( - job_dir, - "run.config.proto", - &NSJAIL_CONFIG_RUN_DENO_CONTENT - .replace("{JOB_DIR}", job_dir) - .replace("{CACHE_DIR}", DENO_CACHE_DIR) - .replace("{CLONE_NEWUSER}", &(!disable_nuser).to_string()), - ) - .await?; - } - - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - "started deno code execution {}", - job.id - ); - let hostname_base = base_url.split("://").last().unwrap_or("localhost"); - let hostname_internal = base_internal_url.split("://").last().unwrap_or("localhost"); - let deno_auth_tokens = format!("{token}@{hostname_base};{token}@{hostname_internal}"); - - let child = if !disable_nsjail { - Command::new(nsjail_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("DENO_AUTH_TOKENS", deno_auth_tokens) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec![ - "--config", - "run.config.proto", - "--", - deno_path, - "run", - "--unstable", - "--v8-flags=--max-heap-size=2048", - "-A", - "/tmp/main.ts", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - } else { - Command::new(deno_path) - .current_dir(job_dir) - .env_clear() - .envs(reserved_variables) - .env("PATH", path_env) - .env("DENO_AUTH_TOKENS", deno_auth_tokens) - .env("BASE_INTERNAL_URL", base_internal_url) - .args(vec![ - "run", - "--unstable", - "--v8-flags=--max-heap-size=2048", - "-A", - "main.ts", - ]) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()? - }; - *status = handle_child(job, db, logs, last_line, timeout, child).await; - tracing::info!( - worker_name = %worker_name, - job_id = %job.id, - workspace_id = %job.workspace_id, - is_ok = status.is_ok(), - "finished deno code execution {}", - job.id - ); + } else { + reserved_variables.insert("PYTHONPATH".to_string(), format!("{job_dir}/dependencies")); } + + tracing::info!( + worker_name = %worker_name, + job_id = %job.id, + workspace_id = %job.workspace_id, + "started python code execution {}", + job.id + ); + let child = if !disable_nsjail { + Command::new(nsjail_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec![ + "--config", + "run.config.proto", + "--", + python_path, + "-u", + "/tmp/main.py", + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + } else { + Command::new(python_path) + .current_dir(job_dir) + .env_clear() + .envs(reserved_variables) + .env("PATH", path_env) + .env("BASE_INTERNAL_URL", base_internal_url) + .args(vec!["-u", "main.py"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()? + }; + + *status = handle_child(&job.id, db, logs, last_line, timeout, child).await; } Ok(()) } +async fn create_dependencies_dir(job_dir: &str) { + DirBuilder::new() + .recursive(true) + .create(&format!("{job_dir}/dependencies")) + .await + .expect("could not create dependencies dir"); +} + async fn handle_dependency_job( job: &QueuedJob, logs: &mut String, job_dir: &String, + status: &mut error::Result, + db: &sqlx::Pool, + last_line: &mut String, + timeout: i32, + go_path: &str, +) -> error::Result<()> { + let content = match job.language { + Some(ScriptLang::Python3) => { + create_dependencies_dir(job_dir).await; + + let requirements = job + .raw_code + .as_ref() + .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?; + logs.push_str(&format!("content of requirements:\n{}\n", &requirements)); + let file = "requirements.in"; + write_file(job_dir, file, &requirements).await?; + let child = Command::new("pip-compile") + .current_dir(job_dir) + .args(vec!["-q", "--no-header", file]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + *status = handle_child(&job.id, db, logs, last_line, timeout, child).await; + if status.is_ok() && status.as_ref().unwrap().success() { + let path_lock = format!("{job_dir}/requirements.txt"); + let mut file = File::open(path_lock).await?; + + let mut req_content = "".to_string(); + file.read_to_string(&mut req_content).await?; + Ok(req_content + .lines() + .filter(|x| !x.trim_start().starts_with('#')) + .map(|x| x.to_string()) + .collect::>() + .join("\n")) + } else { + Err(format!("Lock file generation failed: {status:?}")) + } + } + Some(ScriptLang::Go) => { + let requirements = job + .raw_code + .as_ref() + .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?; + install_go_dependencies( + &job.id, + &requirements, + logs, + job_dir, + status, + db, + last_line, + timeout, + go_path, + false, + ) + .await + .map_err(|e| e.to_string()) + } + _ => Err("Language incompatible with dep job".to_string()), + }; + + match content { + Ok(content) => { + let as_json = json!(content); + + *last_line = + format!(r#"{{ "success": "Successful lock file generation", "lock": {as_json} }}"#); + + sqlx::query!( + "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", + &content, + &job.script_hash.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + } + Err(error) => { + sqlx::query!( + "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", + &format!("{logs}\n{error}"), + &job.script_hash.unwrap_or(ScriptHash(0)).0, + &job.workspace_id + ) + .execute(db) + .await?; + } + } + Ok(()) +} + +async fn install_go_dependencies( + job_id: &Uuid, + code: &str, + logs: &mut String, + job_dir: &String, status: &mut Result, db: &sqlx::Pool, last_line: &mut String, timeout: i32, -) -> Result<(), Error> { - let requirements = job - .raw_code - .as_ref() - .ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?; - logs.push_str(&format!("content of requirements:\n{}\n", &requirements)); - let file = "requirements.in"; - write_file(job_dir, file, &requirements).await?; - let child = Command::new("pip-compile") + go_path: &str, + preview: bool, +) -> error::Result { + gen_go_mymod(code, job_dir).await?; + let child = Command::new("go") .current_dir(job_dir) - .args(vec!["-q", "--no-header", file]) + .args(vec!["mod", "init", "mymod"]) .stdout(Stdio::piped()) .stderr(Stdio::piped()) .spawn()?; - *status = handle_child(job, db, logs, last_line, timeout, child).await; - Ok(if status.is_ok() && status.as_ref().unwrap().success() { - let path_lock = format!("{}/requirements.txt", job_dir); - let mut file = File::open(path_lock).await?; + *status = handle_child(job_id, db, logs, last_line, timeout, child).await; + if status.is_ok() { + let child = Command::new(go_path) + .current_dir(job_dir) + .args(vec!["mod", "tidy"]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn()?; + *status = handle_child(job_id, db, logs, last_line, timeout, child).await; + } + if status.is_ok() && status.as_ref().unwrap().success() { + if preview { + Ok(String::new()) + } else { + let mut req_content = "".to_string(); - let mut content = "".to_string(); - file.read_to_string(&mut content).await?; - content = content - .lines() - .filter(|x| !x.trim_start().starts_with('#')) - .map(|x| x.to_string()) - .collect::>() - .join("\n"); - let as_json = json!(content); + let mut file = File::open(format!("{job_dir}/go.mod")).await?; + file.read_to_string(&mut req_content).await?; - *last_line = - format!(r#"{{ "success": "Successful lock file generation", "lock": {as_json} }}"#); + req_content.push_str(&format!("\n{GO_REQ_SPLITTER}\n")); - sqlx::query!( - "UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3", - &content, - &job.script_hash.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; + let mut file = File::open(format!("{job_dir}/go.sum")).await?; + file.read_to_string(&mut req_content).await?; + + Ok(req_content) + } } else { - sqlx::query!( - "UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3", - &logs.clone(), - &job.script_hash.unwrap_or(ScriptHash(0)).0, - &job.workspace_id - ) - .execute(db) - .await?; - }) + tracing::info!("go mod error"); + + Err(error::Error::ExecutionErr(format!( + "Lock file generation failed. Status: {status:?}", + ))) + } +} + +async fn gen_go_mymod(code: &str, job_dir: &String) -> error::Result<()> { + let code = &format!("package main\n\n{code}"); + + let mymod_dir = format!("{job_dir}/mymod"); + DirBuilder::new() + .recursive(true) + .create(&mymod_dir) + .await + .expect("could not create go's mymod dir"); + + write_file(&mymod_dir, "main.go", &code).await?; + + Ok(()) } async fn get_reserved_variables( @@ -979,13 +1344,14 @@ async fn get_reserved_variables( } async fn handle_child( - job: &QueuedJob, + job_id: &Uuid, db: &DB, logs: &mut String, last_line: &mut String, timeout: i32, mut child: Child, ) -> crate::error::Result { + let job_id = job_id.clone(); let stderr = child .stderr .take() @@ -1026,7 +1392,6 @@ async fn handle_child( }); let (tx, mut rx) = mpsc::channel::(100); - let id = job.id; tokio::spawn(async move { while !done4.load(Ordering::Relaxed) { @@ -1065,12 +1430,12 @@ async fn handle_child( tokio::spawn(async move { while !&done3.load(Ordering::Relaxed) { - let q = sqlx::query!("UPDATE queue SET last_ping = now() WHERE id = $1", id) + let q = sqlx::query!("UPDATE queue SET last_ping = now() WHERE id = $1", job_id) .execute(&db2) .await; if q.is_err() { - tracing::error!("error setting last ping for id {}", id); + tracing::error!("error setting last ping for id {}", job_id); } tokio::time::sleep(Duration::from_secs(5)).await; @@ -1096,18 +1461,18 @@ async fn handle_child( let to_send = logs.chars().skip(start).collect::(); if start != end { - concat_logs(&to_send, id, db).await; + concat_logs(&to_send, &job_id, db).await; start = end; } - let canceled = sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", id) + let canceled = sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", job_id) .fetch_one(db) .await - .map_err(|e| tracing::error!("error getting canceled for id {}: {e}", id)) + .map_err(|e| tracing::error!("error getting canceled for id {}: {e}", job_id)) .unwrap_or(false); if canceled { - tracing::info!("killed after cancel: {}", job.id); + tracing::info!("killed after cancel: {}", job_id); done.store(true, Ordering::Relaxed); } @@ -1119,12 +1484,12 @@ async fn handle_child( canceled_reason = 'duration > {}' WHERE id = $1", timeout )) - .bind(id) + .bind(job_id) .execute(db) .await; if q.is_err() { - tracing::error!("error setting canceled for id {}", id); + tracing::error!("error setting canceled for id {}", job_id); } } last_update = chrono::Utc::now().timestamp_millis(); @@ -1133,7 +1498,7 @@ async fn handle_child( if let Some(nl) = nl { if logs.chars().count() > MAX_LOG_SIZE as usize{ - tracing::info!("Too many logs lines: {}", job.id); + tracing::info!("Too many logs lines: {}", job_id); logs.push_str("Too many logs lines. Limit is 200000 chars. Killing job."); done.store(true, Ordering::Relaxed); } @@ -1144,7 +1509,7 @@ async fn handle_child( *last_line = nl; } else { let to_send = logs.chars().skip(start).collect::(); - concat_logs(&to_send, id, db).await; + concat_logs(&to_send, &job_id, db).await; break; } }, @@ -1171,7 +1536,7 @@ async fn set_logs(logs: &str, id: uuid::Uuid, db: &DB) { }; } -async fn concat_logs(logs: &str, id: uuid::Uuid, db: &DB) { +async fn concat_logs(logs: &str, id: &Uuid, db: &DB) { if sqlx::query!( "UPDATE queue SET logs = concat(logs, $1::text) WHERE id = $2", logs.to_owned(), @@ -1441,6 +1806,32 @@ mod tests { } } + #[sqlx::test(fixtures("base"))] + async fn test_go_job(db: DB) { + initialize_tracing().await; + + let content = r#" +import "fmt" + +func main(derp string) (string, error) { + fmt.Println("Hello, 世界") + return fmt.Sprintf("hello %s", derp), nil +} + "# + .to_owned(); + + let result = RunJob::from(JobPayload::Code(RawCode { + content, + path: None, + language: ScriptLang::Go, + })) + .arg("derp", json!("world")) + .wait_until_complete(&db) + .await; + + assert_eq!(result, serde_json::json!("hello world")); + } + #[sqlx::test(fixtures("base"))] async fn test_python_job(db: DB) { initialize_tracing().await; @@ -2098,6 +2489,7 @@ def main(error, port): base_url: String::new(), disable_nuser: false, disable_nsjail: false, + keep_job_dir: false, }; run_worker( diff --git a/frontend/src/lib/components/Editor.svelte b/frontend/src/lib/components/Editor.svelte index 9a962b687b..e5cbb3b6d0 100644 --- a/frontend/src/lib/components/Editor.svelte +++ b/frontend/src/lib/components/Editor.svelte @@ -47,8 +47,7 @@ let divEl: HTMLDivElement | null = null let editor: monaco.editor.IStandaloneCodeEditor - export let deno = false - export let lang = deno ? 'typescript' : 'python' + export let lang: 'typescript' | 'python' | 'go' export let code: string = '' export let hash: string = randomHash() export let cmdEnterAction: (() => void) | undefined = undefined @@ -139,201 +138,190 @@ export async function reloadWebsocket() { await closeWebsockets() - if (lang == 'python' || deno) { - const { MonacoLanguageClient } = await import('monaco-languageclient') - const { CloseAction, ErrorAction } = await import('vscode-languageclient') - const { toSocket, WebSocketMessageReader, WebSocketMessageWriter } = await import( - 'vscode-ws-jsonrpc' - ) - const vscode = await import('vscode') - const { RequestType } = await import('vscode-jsonrpc') - // install Monaco language client services - const { MonacoServices } = await import('monaco-languageclient') + const { MonacoLanguageClient } = await import('monaco-languageclient') + const { CloseAction, ErrorAction } = await import('vscode-languageclient') + const { toSocket, WebSocketMessageReader, WebSocketMessageWriter } = await import( + 'vscode-ws-jsonrpc' + ) + const vscode = await import('vscode') + const { RequestType } = await import('vscode-jsonrpc') + // install Monaco language client services + const { MonacoServices } = await import('monaco-languageclient') - monacoServices = MonacoServices.install() + monacoServices = MonacoServices.install() - function createLanguageClient( - transports: MessageTransports, - name: string, - initializationOptions?: any - ) { - const client = new MonacoLanguageClient({ - name: name, - clientOptions: { - documentSelector: deno ? ['typescript'] : ['python'], - errorHandler: { - error: () => ({ action: ErrorAction.Continue }), - closed: () => ({ - action: CloseAction.Restart - }) - }, - markdown: { - isTrusted: true - }, - - // workspaceFolder: { uri: Uri.parse(`/tmp/${name}`), name: 'tmp', index: 0 }, - initializationOptions, - middleware: { - workspace: { - configuration: (params, token, configuration) => { - return [ - { - enable: true - } - ] - } - } - } - }, - connectionProvider: { - get: () => { - return Promise.resolve(transports) - } - } - }) - return client - } - - async function connectToLanguageServer(url: string, name: string, options?: any) { - try { - const webSocket = new WebSocket(url) - - webSocket.onopen = async () => { - const socket = toSocket(webSocket) - const reader = new WebSocketMessageReader(socket) - const writer = new WebSocketMessageWriter(socket) - const languageClient = createLanguageClient({ reader, writer }, name, options) - websockets.push([languageClient, webSocket]) - - reader.onClose(async () => { - try { - console.log('CLOSE') - websocketAlive[name] = false - await languageClient.stop() - } catch (err) { - console.error(err) - } + function createLanguageClient( + transports: MessageTransports, + name: string, + initializationOptions?: any + ) { + const client = new MonacoLanguageClient({ + name: name, + clientOptions: { + documentSelector: [lang], + errorHandler: { + error: () => ({ action: ErrorAction.Continue }), + closed: () => ({ + action: CloseAction.Restart }) - socket.onClose((_code, _reason) => { - websocketAlive[name] = false - }) - - try { - console.log('started client') - await languageClient.start() - } catch (err) { - console.log('err at client') - console.error(err) - throw new Error(err) - } - - lastWsAttempt = new Date() - nbWsAttempt = 0 - if (name == 'deno') { - command && command.dispose() - command = undefined - command = vscode.commands.registerCommand( - 'deno.cache', - (uris: DocumentUri[] = []) => { - languageClient.sendRequest(new RequestType('deno/cache'), { - referrer: { uri }, - uris: uris.map((uri) => ({ uri })) - }) - } - ) - } - - websocketAlive[name] = true - } - } catch (err) { - console.error(`connection to ${name} language server failed`) - } - } - - if (deno) { - await connectToLanguageServer(`wss://${$page.url.host}/ws/deno`, 'deno', { - certificateStores: null, - enablePaths: [], - config: null, - importMap: null, - internalDebug: false, - lint: false, - path: null, - tlsCertificate: null, - unsafelyIgnoreCertificateErrors: null, - unstable: true, - enable: true, - cache: null, - codeLens: { - implementations: true, - references: true }, - suggest: { - autoImports: true, - completeFunctionCalls: false, - names: true, - paths: true, - imports: { - autoDiscover: true, - hosts: { - 'https://deno.land': true + markdown: { + isTrusted: true + }, + + // workspaceFolder: { uri: Uri.parse(`/tmp/${name}`), name: 'tmp', index: 0 }, + initializationOptions, + middleware: { + workspace: { + configuration: (params, token, configuration) => { + return [ + { + enable: true + } + ] } } } - }) - } else { - await connectToLanguageServer(`wss://${$page.url.host}/ws/pyright`, 'pyright', { - executionEnvironments: [ - { - root: '/tmp/pyright', - pythonVersion: '3.7', - pythonPlatform: 'platform', - extraPaths: [] - } - ] - }) - - connectToLanguageServer(`wss://${$page.url.host}/ws/black`, 'black', { - formatters: { - black: { - command: 'black', - args: ['--quiet', '-'] - } - }, - formatFiletypes: { - python: 'black' - } - }) - } - - websocketInterval && clearInterval(websocketInterval) - websocketInterval = setInterval(() => { - console.log( - websocketInterval, - document.visibilityState, - new Date().getTime() - lastWsAttempt.getTime(), - nbWsAttempt - ) - if (document.visibilityState == 'visible') { - if ( - !lastWsAttempt || - (new Date().getTime() - lastWsAttempt.getTime() > 60000 && nbWsAttempt < 2) - ) { - if (!websocketAlive.black && !websocketAlive.deno && !websocketAlive.pyright) { - console.log('reconnecting to language servers') - lastWsAttempt = new Date() - nbWsAttempt++ - reloadWebsocket() - } else { - if (nbWsAttempt >= 2) { - sendUserToast('Giving up on establishing smart assistant connection', true) - clearInterval(websocketInterval) - } - } + }, + connectionProvider: { + get: () => { + return Promise.resolve(transports) } } - }, 5000) + }) + return client } + + async function connectToLanguageServer(url: string, name: string, options?: any) { + try { + const webSocket = new WebSocket(url) + + webSocket.onopen = async () => { + const socket = toSocket(webSocket) + const reader = new WebSocketMessageReader(socket) + const writer = new WebSocketMessageWriter(socket) + const languageClient = createLanguageClient({ reader, writer }, name, options) + websockets.push([languageClient, webSocket]) + + reader.onClose(async () => { + try { + console.log('CLOSE') + websocketAlive[name] = false + await languageClient.stop() + } catch (err) { + console.error(err) + } + }) + socket.onClose((_code, _reason) => { + websocketAlive[name] = false + }) + + try { + console.log('started client') + await languageClient.start() + } catch (err) { + console.log('err at client') + console.error(err) + throw new Error(err) + } + + lastWsAttempt = new Date() + nbWsAttempt = 0 + if (name == 'deno') { + command && command.dispose() + command = undefined + command = vscode.commands.registerCommand('deno.cache', (uris: DocumentUri[] = []) => { + languageClient.sendRequest(new RequestType('deno/cache'), { + referrer: { uri }, + uris: uris.map((uri) => ({ uri })) + }) + }) + } + + websocketAlive[name] = true + } + } catch (err) { + console.error(`connection to ${name} language server failed`) + } + } + + if (lang == 'typescript') { + await connectToLanguageServer(`wss://${$page.url.host}/ws/deno`, 'deno', { + certificateStores: null, + enablePaths: [], + config: null, + importMap: null, + internalDebug: false, + lint: false, + path: null, + tlsCertificate: null, + unsafelyIgnoreCertificateErrors: null, + unstable: true, + enable: true, + cache: null, + codeLens: { + implementations: true, + references: true + }, + suggest: { + autoImports: true, + completeFunctionCalls: false, + names: true, + paths: true, + imports: { + autoDiscover: true, + hosts: { + 'https://deno.land': true + } + } + } + }) + } else if (lang === 'python') { + await connectToLanguageServer(`wss://${$page.url.host}/ws/pyright`, 'pyright', { + executionEnvironments: [ + { + root: '/tmp/pyright', + pythonVersion: '3.7', + pythonPlatform: 'platform', + extraPaths: [] + } + ] + }) + + connectToLanguageServer(`wss://${$page.url.host}/ws/black`, 'black', { + formatters: { + black: { + command: 'black', + args: ['--quiet', '-'] + } + }, + formatFiletypes: { + python: 'black' + } + }) + } + + websocketInterval && clearInterval(websocketInterval) + websocketInterval = setInterval(() => { + if (document.visibilityState == 'visible') { + if ( + !lastWsAttempt || + (new Date().getTime() - lastWsAttempt.getTime() > 60000 && nbWsAttempt < 2) + ) { + if (!websocketAlive.black && !websocketAlive.deno && !websocketAlive.pyright) { + console.log('reconnecting to language servers') + lastWsAttempt = new Date() + nbWsAttempt++ + reloadWebsocket() + } else { + if (nbWsAttempt >= 2) { + sendUserToast('Giving up on establishing smart assistant connection', true) + clearInterval(websocketInterval) + } + } + } + } + }, 5000) } async function closeWebsockets() { @@ -388,15 +376,13 @@ editor.onDidFocusEditorText(() => { dispatch('focus') - if (deno || lang == 'typescript') { - if ( - !websocketAlive.black && - !websocketAlive.deno && - !websocketAlive.pyright && - !websocketInterval - ) { - reloadWebsocket() - } + if ( + !websocketAlive.black && + !websocketAlive.deno && + !websocketAlive.pyright && + !websocketInterval + ) { + reloadWebsocket() } }) @@ -404,9 +390,7 @@ dispatch('blur') }) - if (lang == 'python' || deno) { - reloadWebsocket() - } + reloadWebsocket() return () => { try { diff --git a/frontend/src/lib/components/EditorBar.svelte b/frontend/src/lib/components/EditorBar.svelte index 2df3910a97..dfebf91124 100644 --- a/frontend/src/lib/components/EditorBar.svelte +++ b/frontend/src/lib/components/EditorBar.svelte @@ -14,7 +14,7 @@ import ResourceEditor from './ResourceEditor.svelte' import VariableEditor from './VariableEditor.svelte' - export let lang: 'python3' | 'deno' + export let lang: 'python3' | 'deno' | 'go' export let editor: Editor export let websocketAlive: { pyright: boolean; black: boolean; deno: boolean } @@ -25,7 +25,7 @@ let resourceEditor: ResourceEditor let codeViewer: Modal - let codeLang: 'python3' | 'deno' = 'deno' + let codeLang: 'python3' | 'deno' | 'go' = 'deno' let codeContent: string = '' async function loadVariables() { @@ -85,7 +85,7 @@ if (!path) { if (lang == 'deno') { editor.insertAtCursor(`Deno.env.get('${name}')`) - } else { + } else if (lang == 'python3') { if (!editor.getCode().includes('import os')) { editor.insertAtBeginning('import os\n') } @@ -100,7 +100,7 @@ ) } editor.insertAtCursor(`(await wmill.getVariable('${path}'))`) - } else { + } else if (lang == 'python3') { if (!editor.getCode().includes('import wmill')) { editor.insertAtBeginning('import wmill\n') } @@ -139,7 +139,7 @@ ) } editor.insertAtCursor(`(await wmill.getResource('${path}'))`) - } else { + } else if (lang == 'python3') { if (!editor.getCode().includes('import wmill')) { editor.insertAtBeginning('import wmill\n') } diff --git a/frontend/src/lib/components/FlowModulesViewer.svelte b/frontend/src/lib/components/FlowModulesViewer.svelte index 24af8bdbec..31b7a20e1f 100644 --- a/frontend/src/lib/components/FlowModulesViewer.svelte +++ b/frontend/src/lib/components/FlowModulesViewer.svelte @@ -1,15 +1,12 @@ + + diff --git a/frontend/src/lib/components/ModuleStep.svelte b/frontend/src/lib/components/ModuleStep.svelte index ea6637ba3f..d9810212f8 100644 --- a/frontend/src/lib/components/ModuleStep.svelte +++ b/frontend/src/lib/components/ModuleStep.svelte @@ -1,5 +1,5 @@