kalugin66/windmill
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
Ruben Fiszel
2022-05-05 04:25:58 +02:00
commit 2e132878e4
256 changed files with 46832 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
.dockerignore Normal file
View File

@@ -0,0 +1,5 @@
frontend/node_modules/
frontend/build/
frontend/.svelte-kit/
backend/target/

3
.env Normal file
View File

@@ -0,0 +1,3 @@
SITE_URL=localhost
DB_PASSWORD=changeme
POSTGRES_VERSION=13.3.0

7
.github/Dockerfile vendored Normal file
View File

@@ -0,0 +1,7 @@
FROM nikolaik/python-nodejs
RUN npm install -g @apidevtools/swagger-cli
RUN pip install openapi-python-client
RUN pip install poetry

38
.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@@ -0,0 +1,38 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: ''
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior:
1. Go to '...'
2. Click on '....'
3. Scroll down to '....'
4. See error
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**Desktop (please complete the following information):**
- OS: [e.g. iOS]
- Browser [e.g. chrome, safari]
- Version [e.g. 22]
**Smartphone (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Browser [e.g. stock browser, safari]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.

16
.github/change-versions.sh vendored Executable file
View File

@@ -0,0 +1,16 @@
#!/bin/bash
VERSION=$1
echo "Updating versions to: $VERSION"
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" backend/Cargo.toml
sed -i -e "/version: /s/: .*/: $VERSION/" backend/openapi.yaml
sed -i -e "/\"version\": /s/: .*,/: \"$VERSION\",/" frontend/package.json
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill/pyproject.toml
sed -i -e "/^windmill-api =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill/pyproject.toml
sed -i -e "/^version =/s/= .*/= \"$VERSION\"/" python-client/wmill_pg/pyproject.toml
sed -i -e "/^wmill =/s/= .*/= \"\\^$VERSION\"/" python-client/wmill_pg/pyproject.toml
sed -i -e "/^wmill =/s/= .*/= \">=$VERSION\"/" Pipfile
sed -i -e "/^wmill_pg =/s/= .*/= \">=$VERSION\"/" Pipfile
sed -i -zE "s/name = \"windmill\"\nversion = \"[^\"]*\"\\n(.*)/name = \"windmill\"\nversion = \"$VERSION\"\\n\\1/" backend/Cargo.lock

14
.github/workflows/change-versions.yml vendored Normal file
View File

@@ -0,0 +1,14 @@
name: Change versions
on:
pull_request:
types: [opened, synchronize, reopened]
paths:
- "version.txt"
jobs:
change_version:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Change versions
run: ./.github/change-versions.sh "$(cat version.txt)"
- uses: stefanzweifel/git-auto-commit-action@v4

13
.github/workflows/clean-docker.yml vendored Normal file
View File

@@ -0,0 +1,13 @@
name: Clean docker
on:
schedule:
# * is a special character in YAML so you have to quote this string
- cron: "0 0 */2 * *"
jobs:
build:
runs-on: [self-hosted, new]
steps:
- name: clean docker
run: |
sudo docker system prune -f

18
.github/workflows/deploy_to_windmill.yml vendored Normal file
View File

@@ -0,0 +1,18 @@
name: Deploy to windmill.dev
on:
push:
branches: [main]
jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- name: Deploy to windmill.dev
uses: windmill-labs/windmill-gh-action-deploy@v1.0.0
with:
dry_run: false
input_dir: community
windmill_workspace: starter
windmill_token: ${{ secrets.WINDMILL_API_TOKEN }}

39
.github/workflows/docker-image.yml vendored Normal file
View File

@@ -0,0 +1,39 @@
name: Docker Image CI
on:
push:
branches: [main]
pull_request:
types: [opened, synchronize, reopened]
concurrency:
group: ${{ github.ref }}
cancel-in-progress: true
jobs:
build:
runs-on: [self-hosted, new]
env:
DOCKER_BUILDKIT: 1
steps:
# - name: Wait for release to succeed
# if: github.ref == 'refs/heads/main'
# uses: lewagon/wait-on-check-action@v1.0.0
# with:
# ref: ${{ github.ref }}
# check-name: "Release please"
# repo-token: ${{ secrets.GITHUB_TOKEN }}
# wait-interval: 10
- uses: actions/checkout@v2
with:
fetch-depth: 0
- name: deploy staging stack
run: |
docker build . --cache-from "registry.wimill.xyz/windmill:staging" -t "registry.wimill.xyz/windmill:staging" --build-arg BUILDKIT_INLINE_CACHE=1
docker push "registry.wimill.xyz/windmill:staging"
- name: deploy demo stack
if: github.ref == 'refs/heads/main'
run: |
docker tag registry.wimill.xyz/windmill:staging registry.wimill.xyz/windmill:main
docker push registry.wimill.xyz/windmill:main
# - name: pruning unused images
# run: sudo docker image prune -a

38
.github/workflows/on-release.yml vendored Normal file
View File

@@ -0,0 +1,38 @@
name: Build LSP Docker
on:
push:
branches: [main]
paths:
- "python-client/**"
- "Pipfile"
- ".github/workflows/on-release.yml"
jobs:
build_lsp:
runs-on: [self-hosted, new]
steps:
- name: Wait for release to succeed
if: github.ref == 'refs/heads/main'
uses: lewagon/wait-on-check-action@v1.0.0
with:
ref: ${{ github.ref }}
check-name: "Release please"
repo-token: ${{ secrets.GITHUB_TOKEN }}
wait-interval: 10
- uses: actions/checkout@v2
- name: Upload python client
env:
PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
run: |
cd python-client
export PATH=$PATH:/usr/local/bin
export PATH=$PATH:/root/.local/bin
./publish.sh
- name: Build the Docker image
run: |
echo "branch main"
sudo docker pull "registry.wimill.xyz/lsp:main" || true
sudo docker build -f DockerfileLSP . --cache-from "registry.wimill.xyz/lsp:main" -t "registry.wimill.xyz/lsp:main" --build-arg BUILDKIT_INLINE_CACHE=1
- name: push to registry
run: |
sudo docker push "registry.wimill.xyz/lsp:main"

15
.github/workflows/release-please.yml vendored Normal file
View File

@@ -0,0 +1,15 @@
on:
push:
branches:
- main
name: release-please
jobs:
release-please:
name: "Release please"
runs-on: ubuntu-latest
steps:
- uses: GoogleCloudPlatform/release-please-action@v2
with:
release-type: simple
package-name: windmill
token: ${{ secrets.PAT_TOKEN }}

5
.gitignore vendored Normal file
View File

@@ -0,0 +1,5 @@
target/
.DS_Store
local/
frontend/src/routes/test.svelte
CaddyfileRemoteMalo

72
CHANGELOG.md Normal file
View File

@@ -0,0 +1,72 @@
# Changelog
## [1.5.0](https://www.github.com/windmill-labs/windmill-server/compare/v1.4.2...v1.5.0) (2022-05-02)
### Features
* dynamic input ([2aac191](https://www.github.com/windmill-labs/windmill-server/commit/2aac1917c295f478c5d81ed0346857385e591ceb))
### [1.4.2](https://www.github.com/windmill-labs/windmill-server/compare/v1.4.1...v1.4.2) (2022-04-29)
### Bug Fixes
* fix release ([13f8e39](https://www.github.com/windmill-labs/windmill-server/commit/13f8e39a11025b82c45dd16092df438f19d910e5))
### [1.4.1](https://www.github.com/windmill-labs/windmill-server/compare/v1.4.0...v1.4.1) (2022-04-29)
### Bug Fixes
* base url python client ([b221bf0](https://www.github.com/windmill-labs/windmill-server/commit/b221bf01e30155007a0dcaac2799cc762a7b5f3b))
## [1.4.0](https://www.github.com/windmill-labs/windmill-server/compare/v1.3.0...v1.4.0) (2022-04-27)
### Features
* variables backend logic ([9864028](https://www.github.com/windmill-labs/windmill-server/commit/9864028c648cec368f4541145e8b21e10d81627b))
* variables backend logic ([d762f93](https://www.github.com/windmill-labs/windmill-server/commit/d762f93a0c75ad73229fd7a56ae3372ff2e8e41a))
* variables backend logic ([3e567a8](https://www.github.com/windmill-labs/windmill-server/commit/3e567a8782d0377afde9a362d4bea6dff6cd5b3f))
## [1.3.0](https://www.github.com/windmill-labs/windmill-server/compare/v1.2.0...v1.3.0) (2022-04-27)
### Features
* secret decryption is audited ([d5c5877](https://www.github.com/windmill-labs/windmill-server/commit/d5c58771e1ce0acf766843bc38d724044c905569))
## [1.2.0](https://www.github.com/windmill-labs/windmill-server/compare/v1.1.1...v1.2.0) (2022-04-15)
### Features
* custom env ([a89e807](https://www.github.com/windmill-labs/windmill-server/commit/a89e807aa4f5ec3584285a1e9ef126a0a77cf766))
### [1.1.1](https://www.github.com/windmill-labs/windmill-server/compare/v1.1.0...v1.1.1) (2022-03-14)
### Bug Fixes
* pg commit ([35967d5](https://www.github.com/windmill-labs/windmill-server/commit/35967d50461a5a53ca3d975e908f74ae54c02100))
## [1.1.0](https://www.github.com/windmill-labs/windmill-server/compare/v1.0.0...v1.1.0) (2022-03-13)
### Features
* query_pg is in integrated into wmill ([2e36244](https://www.github.com/windmill-labs/windmill-server/commit/2e3624447216009e1e8ec1ba416952697a0ef4d4))
## 1.0.0 (2022-02-06)
### Features
* functioning workspaces ([38de8c3](https://www.github.com/windmill-labs/windmill/commit/38de8c3f572f3bd3b8a3252079d930572723ab8a))
## [0.9.0](https://www.github.com/windmill-labs/windmill/compare/v0.0.1...v0.9.0) (2021-12-30)
### Features
- Renaming to windmill

4
Caddyfile Normal file
View File

@@ -0,0 +1,4 @@
{$SITE_URL} {
bind {$ADDRESS}
reverse_proxy /* server:8000
}

99
Dockerfile Normal file
View File

@@ -0,0 +1,99 @@
FROM python:3.10-slim-buster as nsjail
WORKDIR /nsjail
RUN apt-get -y update \
&& apt-get install -y \
bison=2:3.3.* \
flex=2.6.* \
g++=4:8.3.* \
gcc=4:8.3.* \
git=1:2.20.* \
libprotobuf-dev=3.6.* \
libnl-route-3-dev=3.4.* \
make=4.2.* \
pkg-config=0.29-6 \
protobuf-compiler=3.6.*
RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \
&& git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800
RUN make
FROM mhart/alpine-node:14 as frontend
# install dependencies
WORKDIR /frontend
COPY ./frontend/package.json ./frontend/package-lock.json ./
RUN npm ci
# Copy all local files into the image.
COPY frontend .
RUN mkdir /backend
COPY /backend/openapi.yaml /backend/openapi.yaml
RUN npm run generate-backend-client
RUN npm run build
FROM rust:slim-buster as builder
RUN apt-get update && apt-get install -y git libssl-dev pkg-config
RUN USER=root cargo new --bin windmill
WORKDIR /windmill
COPY ./backend/Cargo.toml .
COPY ./backend/Cargo.lock .
COPY ./backend/.cargo/ .cargo/
RUN apt-get -y update \
&& apt-get install -y \
curl
ENV CARGO_INCREMENTAL=1
RUN cargo build --release
RUN rm src/*.rs
RUN rm ./target/release/deps/windmill*
ENV SQLX_OFFLINE=true
ADD ./backend ./
ADD ./nsjail /nsjail
COPY --from=1 /frontend /frontend
ADD .git/ .git/
RUN cargo build --release
FROM debian:buster-slim
ARG APP=/usr/src/app
RUN apt-get update \
&& apt-get install -y ca-certificates tzdata libpq5 python3 python3-pip \
make build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev \
libsqlite3-dev wget curl llvm libncurses5-dev libncursesw5-dev xz-utils tk-dev libxml2-dev \
libxmlsec1-dev libffi-dev liblzma-dev mecab-ipadic-utf8 libgdbm-dev libc6-dev git libprotobuf-dev=3.6.* libnl-route-3-dev=3.4.* \
libv8-dev \
&& rm -rf /var/lib/apt/lists/*
ENV TZ=Etc/UTC
ENV PYTHON_VERSION 3.10.4
RUN wget https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz \
&& tar -xf Python-${PYTHON_VERSION}.tgz && cd Python-${PYTHON_VERSION}/ && ./configure --enable-optimizations \
&& make -j 4 && make install
RUN python3 -m pip install pip-tools
COPY --from=builder /windmill/target/release/windmill ${APP}/windmill
COPY --from=nsjail /nsjail/nsjail /bin/nsjail
RUN mkdir -p ${APP}
WORKDIR ${APP}
EXPOSE 8000
CMD ["./windmill"]

16
DockerfileLSP Normal file
View File

@@ -0,0 +1,16 @@
FROM nikolaik/python-nodejs
RUN yarn global add diagnostic-languageserver
RUN yarn global add pyright
RUN pip3 install black tornado python-lsp-jsonrpc
COPY Pipfile .
RUN cat Pipfile
RUN pipenv install
COPY pyls_launcher.py .
CMD ["python3" ,"pyls_launcher.py"]

12
LICENSE Normal file
View File

@@ -0,0 +1,12 @@
Source code in this repository is variously licensed under the Apache License
Version 2.0 (see file ./LICENSE-APACHE),or the AGPLv3 License (see file ./LICENSE-AGPL)
Every file is under copyright (c) Ruben Fiszel 2021 unless otherwise specified.
Every file is under License AGPL unless otherwise specified
or belonging to one of the below cases:
The files under backend/ are AGPL Licensed.
The files under frontend/ are AGPL Licensed.
The files under python-client/ are Apache 2.0 Licensed.
The files under community/ are Apache 2.0 Licensed.

661
LICENSE-AGPL Normal file
View File

@@ -0,0 +1,661 @@
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
Everyone is permitted to copy and distribute verbatim copies
of this license document, but changing it is not allowed.
Preamble
The GNU Affero General Public License is a free, copyleft license for
software and other kinds of works, specifically designed to ensure
cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed
to take away your freedom to share and change the works. By contrast,
our General Public Licenses are intended to guarantee your freedom to
share and change all versions of a program--to make sure it remains free
software for all its users.
When we speak of free software, we are referring to freedom, not
price. Our General Public Licenses are designed to make sure that you
have the freedom to distribute copies of free software (and charge for
them if you wish), that you receive source code or can get it if you
want it, that you can change the software or use pieces of it in new
free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights
with two steps: (1) assert copyright on the software, and (2) offer
you this License which gives you legal permission to copy, distribute
and/or modify the software.
A secondary benefit of defending all users' freedom is that
improvements made in alternate versions of the program, if they
receive widespread use, become available for other developers to
incorporate. Many developers of free software are heartened and
encouraged by the resulting cooperation. However, in the case of
software used on network servers, this result may fail to come about.
The GNU General Public License permits making a modified version and
letting the public access it on a server without ever releasing its
source code to the public.
The GNU Affero General Public License is designed specifically to
ensure that, in such cases, the modified source code becomes available
to the community. It requires the operator of a network server to
provide the source code of the modified version running there to the
users of that server. Therefore, public use of a modified version, on
a publicly accessible server, gives the public access to the source
code of the modified version.
An older license, called the Affero General Public License and
published by Affero, was designed to accomplish similar goals. This is
a different license, not a version of the Affero GPL, but Affero has
released a new version of the Affero GPL which permits relicensing under
this license.
The precise terms and conditions for copying, distribution and
modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of
works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this
License. Each licensee is addressed as "you". "Licensees" and
"recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work
in a fashion requiring copyright permission, other than the making of an
exact copy. The resulting work is called a "modified version" of the
earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based
on the Program.
To "propagate" a work means to do anything with it that, without
permission, would make you directly or secondarily liable for
infringement under applicable copyright law, except executing it on a
computer or modifying a private copy. Propagation includes copying,
distribution (with or without modification), making available to the
public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other
parties to make or receive copies. Mere interaction with a user through
a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices"
to the extent that it includes a convenient and prominently visible
feature that (1) displays an appropriate copyright notice, and (2)
tells the user that there is no warranty for the work (except to the
extent that warranties are provided), that licensees may convey the
work under this License, and how to view a copy of this License. If
the interface presents a list of user commands or options, such as a
menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work
for making modifications to it. "Object code" means any non-source
form of a work.
A "Standard Interface" means an interface that either is an official
standard defined by a recognized standards body, or, in the case of
interfaces specified for a particular programming language, one that
is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other
than the work as a whole, that (a) is included in the normal form of
packaging a Major Component, but which is not part of that Major
Component, and (b) serves only to enable use of the work with that
Major Component, or to implement a Standard Interface for which an
implementation is available to the public in source code form. A
"Major Component", in this context, means a major essential component
(kernel, window system, and so on) of the specific operating system
(if any) on which the executable work runs, or a compiler used to
produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all
the source code needed to generate, install, and (for an executable
work) run the object code and to modify the work, including scripts to
control those activities. However, it does not include the work's
System Libraries, or general-purpose tools or generally available free
programs which are used unmodified in performing those activities but
which are not part of the work. For example, Corresponding Source
includes interface definition files associated with source files for
the work, and the source code for shared libraries and dynamically
linked subprograms that the work is specifically designed to require,
such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users
can regenerate automatically from other parts of the Corresponding
Source.
The Corresponding Source for a work in source code form is that
same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of
copyright on the Program, and are irrevocable provided the stated
conditions are met. This License explicitly affirms your unlimited
permission to run the unmodified Program. The output from running a
covered work is covered by this License only if the output, given its
content, constitutes a covered work. This License acknowledges your
rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not
convey, without conditions so long as your license otherwise remains
in force. You may convey covered works to others for the sole purpose
of having them make modifications exclusively for you, or provide you
with facilities for running those works, provided that you comply with
the terms of this License in conveying all material for which you do
not control copyright. Those thus making or running the covered works
for you must do so exclusively on your behalf, under your direction
and control, on terms that prohibit them from making any copies of
your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under
the conditions stated below. Sublicensing is not allowed; section 10
makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological
measure under any applicable law fulfilling obligations under article
11 of the WIPO copyright treaty adopted on 20 December 1996, or
similar laws prohibiting or restricting circumvention of such
measures.
When you convey a covered work, you waive any legal power to forbid
circumvention of technological measures to the extent such circumvention
is effected by exercising rights under this License with respect to
the covered work, and you disclaim any intention to limit operation or
modification of the work as a means of enforcing, against the work's
users, your or third parties' legal rights to forbid circumvention of
technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you
receive it, in any medium, provided that you conspicuously and
appropriately publish on each copy an appropriate copyright notice;
keep intact all notices stating that this License and any
non-permissive terms added in accord with section 7 apply to the code;
keep intact all notices of the absence of any warranty; and give all
recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey,
and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to
produce it from the Program, in the form of source code under the
terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified
it, and giving a relevant date.
b) The work must carry prominent notices stating that it is
released under this License and any conditions added under section
7. This requirement modifies the requirement in section 4 to
"keep intact all notices".
c) You must license the entire work, as a whole, under this
License to anyone who comes into possession of a copy. This
License will therefore apply, along with any applicable section 7
additional terms, to the whole of the work, and all its parts,
regardless of how they are packaged. This License gives no
permission to license the work in any other way, but it does not
invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display
Appropriate Legal Notices; however, if the Program has interactive
interfaces that do not display Appropriate Legal Notices, your
work need not make them do so.
A compilation of a covered work with other separate and independent
works, which are not by their nature extensions of the covered work,
and which are not combined with it such as to form a larger program,
in or on a volume of a storage or distribution medium, is called an
"aggregate" if the compilation and its resulting copyright are not
used to limit the access or legal rights of the compilation's users
beyond what the individual works permit. Inclusion of a covered work
in an aggregate does not cause this License to apply to the other
parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms
of sections 4 and 5, provided that you also convey the
machine-readable Corresponding Source under the terms of this License,
in one of these ways:
a) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by the
Corresponding Source fixed on a durable physical medium
customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product
(including a physical distribution medium), accompanied by a
written offer, valid for at least three years and valid for as
long as you offer spare parts or customer support for that product
model, to give anyone who possesses the object code either (1) a
copy of the Corresponding Source for all the software in the
product that is covered by this License, on a durable physical
medium customarily used for software interchange, for a price no
more than your reasonable cost of physically performing this
conveying of source, or (2) access to copy the
Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the
written offer to provide the Corresponding Source. This
alternative is allowed only occasionally and noncommercially, and
only if you received the object code with such an offer, in accord
with subsection 6b.
d) Convey the object code by offering access from a designated
place (gratis or for a charge), and offer equivalent access to the
Corresponding Source in the same way through the same place at no
further charge. You need not require recipients to copy the
Corresponding Source along with the object code. If the place to
copy the object code is a network server, the Corresponding Source
may be on a different server (operated by you or a third party)
that supports equivalent copying facilities, provided you maintain
clear directions next to the object code saying where to find the
Corresponding Source. Regardless of what server hosts the
Corresponding Source, you remain obligated to ensure that it is
available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided
you inform other peers where the object code and Corresponding
Source of the work are being offered to the general public at no
charge under subsection 6d.
A separable portion of the object code, whose source code is excluded
from the Corresponding Source as a System Library, need not be
included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any
tangible personal property which is normally used for personal, family,
or household purposes, or (2) anything designed or sold for incorporation
into a dwelling. In determining whether a product is a consumer product,
doubtful cases shall be resolved in favor of coverage. For a particular
product received by a particular user, "normally used" refers to a
typical or common use of that class of product, regardless of the status
of the particular user or of the way in which the particular user
actually uses, or expects or is expected to use, the product. A product
is a consumer product regardless of whether the product has substantial
commercial, industrial or non-consumer uses, unless such uses represent
the only significant mode of use of the product.
"Installation Information" for a User Product means any methods,
procedures, authorization keys, or other information required to install
and execute modified versions of a covered work in that User Product from
a modified version of its Corresponding Source. The information must
suffice to ensure that the continued functioning of the modified object
code is in no case prevented or interfered with solely because
modification has been made.
If you convey an object code work under this section in, or with, or
specifically for use in, a User Product, and the conveying occurs as
part of a transaction in which the right of possession and use of the
User Product is transferred to the recipient in perpetuity or for a
fixed term (regardless of how the transaction is characterized), the
Corresponding Source conveyed under this section must be accompanied
by the Installation Information. But this requirement does not apply
if neither you nor any third party retains the ability to install
modified object code on the User Product (for example, the work has
been installed in ROM).
The requirement to provide Installation Information does not include a
requirement to continue to provide support service, warranty, or updates
for a work that has been modified or installed by the recipient, or for
the User Product in which it has been modified or installed. Access to a
network may be denied when the modification itself materially and
adversely affects the operation of the network or violates the rules and
protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided,
in accord with this section must be in a format that is publicly
documented (and with an implementation available to the public in
source code form), and must require no special password or key for
unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this
License by making exceptions from one or more of its conditions.
Additional permissions that are applicable to the entire Program shall
be treated as though they were included in this License, to the extent
that they are valid under applicable law. If additional permissions
apply only to part of the Program, that part may be used separately
under those permissions, but the entire Program remains governed by
this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option
remove any additional permissions from that copy, or from any part of
it. (Additional permissions may be written to require their own
removal in certain cases when you modify the work.) You may place
additional permissions on material, added by you to a covered work,
for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you
add to a covered work, you may (if authorized by the copyright holders of
that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the
terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or
author attributions in that material or in the Appropriate Legal
Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or
requiring that modified versions of such material be marked in
reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or
authors of the material; or
e) Declining to grant rights under trademark law for use of some
trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that
material by anyone who conveys the material (or modified versions of
it) with contractual assumptions of liability to the recipient, for
any liability that these contractual assumptions directly impose on
those licensors and authors.
All other non-permissive additional terms are considered "further
restrictions" within the meaning of section 10. If the Program as you
received it, or any part of it, contains a notice stating that it is
governed by this License along with a term that is a further
restriction, you may remove that term. If a license document contains
a further restriction but permits relicensing or conveying under this
License, you may add to a covered work material governed by the terms
of that license document, provided that the further restriction does
not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you
must place, in the relevant source files, a statement of the
additional terms that apply to those files, or a notice indicating
where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the
form of a separately written license, or stated as exceptions;
the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly
provided under this License. Any attempt otherwise to propagate or
modify it is void, and will automatically terminate your rights under
this License (including any patent licenses granted under the third
paragraph of section 11).
However, if you cease all violation of this License, then your
license from a particular copyright holder is reinstated (a)
provisionally, unless and until the copyright holder explicitly and
finally terminates your license, and (b) permanently, if the copyright
holder fails to notify you of the violation by some reasonable means
prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is
reinstated permanently if the copyright holder notifies you of the
violation by some reasonable means, this is the first time you have
received notice of violation of this License (for any work) from that
copyright holder, and you cure the violation prior to 30 days after
your receipt of the notice.
Termination of your rights under this section does not terminate the
licenses of parties who have received copies or rights from you under
this License. If your rights have been terminated and not permanently
reinstated, you do not qualify to receive new licenses for the same
material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or
run a copy of the Program. Ancillary propagation of a covered work
occurring solely as a consequence of using peer-to-peer transmission
to receive a copy likewise does not require acceptance. However,
nothing other than this License grants you permission to propagate or
modify any covered work. These actions infringe copyright if you do
not accept this License. Therefore, by modifying or propagating a
covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically
receives a license from the original licensors, to run, modify and
propagate that work, subject to this License. You are not responsible
for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an
organization, or substantially all assets of one, or subdividing an
organization, or merging organizations. If propagation of a covered
work results from an entity transaction, each party to that
transaction who receives a copy of the work also receives whatever
licenses to the work the party's predecessor in interest had or could
give under the previous paragraph, plus a right to possession of the
Corresponding Source of the work from the predecessor in interest, if
the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the
rights granted or affirmed under this License. For example, you may
not impose a license fee, royalty, or other charge for exercise of
rights granted under this License, and you may not initiate litigation
(including a cross-claim or counterclaim in a lawsuit) alleging that
any patent claim is infringed by making, using, selling, offering for
sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this
License of the Program or a work on which the Program is based. The
work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims
owned or controlled by the contributor, whether already acquired or
hereafter acquired, that would be infringed by some manner, permitted
by this License, of making, using, or selling its contributor version,
but do not include claims that would be infringed only as a
consequence of further modification of the contributor version. For
purposes of this definition, "control" includes the right to grant
patent sublicenses in a manner consistent with the requirements of
this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free
patent license under the contributor's essential patent claims, to
make, use, sell, offer for sale, import and otherwise run, modify and
propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express
agreement or commitment, however denominated, not to enforce a patent
(such as an express permission to practice a patent or covenant not to
sue for patent infringement). To "grant" such a patent license to a
party means to make such an agreement or commitment not to enforce a
patent against the party.
If you convey a covered work, knowingly relying on a patent license,
and the Corresponding Source of the work is not available for anyone
to copy, free of charge and under the terms of this License, through a
publicly available network server or other readily accessible means,
then you must either (1) cause the Corresponding Source to be so
available, or (2) arrange to deprive yourself of the benefit of the
patent license for this particular work, or (3) arrange, in a manner
consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have
actual knowledge that, but for the patent license, your conveying the
covered work in a country, or your recipient's use of the covered work
in a country, would infringe one or more identifiable patents in that
country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or
arrangement, you convey, or propagate by procuring conveyance of, a
covered work, and grant a patent license to some of the parties
receiving the covered work authorizing them to use, propagate, modify
or convey a specific copy of the covered work, then the patent license
you grant is automatically extended to all recipients of the covered
work and works based on it.
A patent license is "discriminatory" if it does not include within
the scope of its coverage, prohibits the exercise of, or is
conditioned on the non-exercise of one or more of the rights that are
specifically granted under this License. You may not convey a covered
work if you are a party to an arrangement with a third party that is
in the business of distributing software, under which you make payment
to the third party based on the extent of your activity of conveying
the work, and under which the third party grants, to any of the
parties who would receive the covered work from you, a discriminatory
patent license (a) in connection with copies of the covered work
conveyed by you (or copies made from those copies), or (b) primarily
for and in connection with specific products or compilations that
contain the covered work, unless you entered into that arrangement,
or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting
any implied license or other defenses to infringement that may
otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or
otherwise) that contradict the conditions of this License, they do not
excuse you from the conditions of this License. If you cannot convey a
covered work so as to satisfy simultaneously your obligations under this
License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you
to collect a royalty for further conveying from those to whom you convey
the Program, the only way you could satisfy both those terms and this
License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the
Program, your modified version must prominently offer all users
interacting with it remotely through a computer network (if your version
supports such interaction) an opportunity to receive the Corresponding
Source of your version by providing access to the Corresponding Source
from a network server at no charge, through some standard or customary
means of facilitating copying of software. This Corresponding Source
shall include the Corresponding Source for any work covered by version 3
of the GNU General Public License that is incorporated pursuant to the
following paragraph.
Notwithstanding any other provision of this License, you have
permission to link or combine any covered work with a work licensed
under version 3 of the GNU General Public License into a single
combined work, and to convey the resulting work. The terms of this
License will continue to apply to the part which is the covered work,
but the work with which it is combined will remain governed by version
3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of
the GNU Affero General Public License from time to time. Such new versions
will be similar in spirit to the present version, but may differ in detail to
address new problems or concerns.
Each version is given a distinguishing version number. If the
Program specifies that a certain numbered version of the GNU Affero General
Public License "or any later version" applies to it, you have the
option of following the terms and conditions either of that numbered
version or of any later version published by the Free Software
Foundation. If the Program does not specify a version number of the
GNU Affero General Public License, you may choose any version ever published
by the Free Software Foundation.
If the Program specifies that a proxy can decide which future
versions of the GNU Affero General Public License can be used, that proxy's
public statement of acceptance of a version permanently authorizes you
to choose that version for the Program.
Later license versions may give you additional or different
permissions. However, no additional obligations are imposed on any
author or copyright holder as a result of your choosing to follow a
later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided
above cannot be given local legal effect according to their terms,
reviewing courts shall apply local law that most closely approximates
an absolute waiver of all civil liability in connection with the
Program, unless a warranty or assumption of liability accompanies a
copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest
possible use to the public, the best way to achieve this is to make it
free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest
to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as published
by the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer
network, you should also make sure that it provides a way for users to
get its source. For example, if your program is a web application, its
interface could display a "Source" link that leads users to an archive
of the code. There are many ways you could offer source, and different
solutions will be better for different programs; see section 13 for the
specific requirements.
You should also get your employer (if you work as a programmer) or school,
if any, to sign a "copyright disclaimer" for the program, if necessary.
For more information on this, and how to apply and follow the GNU AGPL, see
<https://www.gnu.org/licenses/>.

201
LICENSE-APACHE Normal file
View File

@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright 2021 Ruben Fiszel
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.

13
NOTICE Normal file
View File

@@ -0,0 +1,13 @@
Ruben Fiszel
Copyright (c) 2021 Ruben Fiszel
Source code in this repository is variously licensed under the Apache License
Version 2.0 or the GNU Affero General Public License. Please see
LICENSE for more information.
* For a copy of the Apache License Version 2.0, please see LICENSE-APACHE
as included in this repository's top-level directory.
* For a copy of the GNU Affero General Public License, please see LICENSE-ALPH
as included in this repository's top-level directory.

23
Pipfile Normal file
View File

@@ -0,0 +1,23 @@
[[source]]
url = "https://pypi.org/simple"
verify_ssl = true
name = "pypi"
[packages]
wmill = ">=1.5.0"
wmill_pg = ">=1.5.0"
requests = "*"
sendgrid = "*"
psycopg2-binary = "*"
mysql-connector-python = "*"
pymongo = "*"
slack_sdk = "*"
google-api-python-client = "*"
pandas = "*"
numpy = "*"
seaborn = "*"
yfinance = "*"
pyowm = "*"
pyairtable = "*"
matplotlib = "*"

74
README.md Normal file
View File

@@ -0,0 +1,74 @@
<p align="center">
<a href="https://alpha.windmill.dev"><img src="./windmill.svg" alt="windmill.dev"></a>
</p>
<p align="center">
<em>Windmill.dev is an OSS developer platform to quickly build production-grade multi-steps automations and internal apps from minimal Python and Typescript scripts.</em>
</p>
<p align="center">
<a href="https://pypi.org/project/wmill" target="_blank">
<img src="https://img.shields.io/pypi/v/wmill?color=%2334D058&label=pypi%20package" alt="Package version">
</a>
<a href="https://discord.gg/V7PM2YHsPB" target="_blank">
<img src="https://discordapp.com/api/guilds/930051556043276338/widget.png" alt="Discord Shield"/>
</a>
</p>
---
**Join the alpha (personal workspaces are free forever)**:
<https://alpha.windmill.dev>
**Documentation**: <https://docs.windmill.dev>
**Discord**: <https://discord.gg/V7PM2YHsPB>
**We are hiring**: Software Engineers, DevOps, Solutions Engineers, Growth:
<https://docs.windmill.dev/hiring>
If you would like to, you can show your support for the project by starring this
repo.
---
# Windmill
![Windmill](./windmill.webp)
Windmill is fully open-sourced:
- community parts and python-client are Apache 2.0
- backend, frontend and everything else under AGPLv3.
## Stack
- postgres as the database
- backend in Rust with the follwing highly-available and horizontally scalable
architecture:
- stateless API backend
- workers that pull jobs from a queue
- frontend in svelte
- scripts executions are sandboxed using google's nsjail
- javascript runtime is deno_core rust library (which itself uses the rusty_v8
and hence V8 underneath)
- typescript runtime is deno
- python runtime is python3
### Developent stack
- caddy is the reverse proxy + handle https
## How to self-host
Complete instructions coming soon
## Copyright
2021 [Ruben Fiszel](https://github.com/rubenfiszel)
## Acknowledgement
This project is inspired from a previous project called
[Delightool](https://github.com/windmill-labs/delightool-legacy) which was also
build by [Ruben](https://github.com/rubenfiszel) but the frontend was realized
in large parts by [Malo Marrec]((https://github.com/malomarrec). Windmill is a
child but distinct project and realized with Malo's blessing.

3
backend/.cargo/config Normal file
View File

@@ -0,0 +1,3 @@
[build]
rustflags = ["--cfg", "tokio_unstable"]
incremental = true

2
backend/.gitignore vendored Normal file
View File

@@ -0,0 +1,2 @@
target/
.env

4170
backend/Cargo.lock generated Normal file
View File

File diff suppressed because it is too large Load Diff

62
backend/Cargo.toml Normal file
View File

@@ -0,0 +1,62 @@
[package]
name = "windmill"
version = "1.5.0"
authors = ["Ruben Fiszel <ruben@rubenfiszel.com>"]
edition = "2021"
[build-dependencies]
deno_core = "^0"
[dependencies]
axum = { version = "^0", features = ["headers"] }
headers = "^0"
hyper = { version = "^0", features = ["full"] }
tokio = { version = "^1", features = ["full", "tracing"] }
tower = "^0"
tower-http = { version = "^0", features = ["trace"] }
tower-cookies = "^0"
serde = "^1"
serde_json = { version = "^1", features = ["preserve_order"] }
uuid = { version = "^0", features = ["serde", "v4"] }
thiserror = "^1"
anyhow = "^1"
chrono = { version = "^0", features = ["serde"]}
tracing = "^0"
tracing-subscriber = { version = "^0", features = ["env-filter", "json"]}
console-subscriber = "^0"
rust-embed = "^6"
mime_guess = "^2"
hex = "^0"
sql-builder = "^3"
argon2 = "^0"
retainer = "^0"
rand = "^0.8.4"
rand_core = { version = "^0.6.3", features = ["std"] }
magic-crypt = "^3"
git-version = "^0"
rustpython-parser = "^0"
cron = "^0"
external-ip = "^4"
lettre = { version = "^0.10.0-rc.4", features = ["rustls-tls", "tokio1", "tokio1-rustls-tls", "builder", "smtp-transport"], default-features = false}
urlencoding = "^2"
oauth2 = "^4"
url = "^2"
reqwest = { version = "^0", features = ["json"] }
time = "0.3.7"
slack-http-verifier = "^0"
serde_urlencoded = "^0"
tokio-tar = "^0"
tempfile = "^3"
tokio-util = { version = "0.7.0", features = ["io"] }
json-pointer = "^0"
itertools = "^0"
regex = "^1"
deno_core = "^0"
indexmap = "~1.6.2"
async-recursion = "^1"
sqlx = { version = "^0", features = ["macros", "offline", "migrate", "uuid", "json", "chrono", "postgres", "runtime-tokio-rustls"]}
dotenv = "^0"
ulid = { version = "^0", features = ["uuid"] }
futures = "^0"

95
backend/LICENSE Normal file
View File

@@ -0,0 +1,95 @@
Business Source License 1.1
Parameters
Licensor: Ruben Fiszel
Licensed Work: windmill.dev backend 0.9.0
The Licensed Work is (c) 2021 Ruben Fiszel
Additional Use Grant: None
Change Date: 2026-01-01
Change License: Apache License, Version 2.0
For information about alternative licensing arrangements for the Software,
please visit: https://windmill.dev
Notice
The Business Source License (this document, or the “License”) is not an Open
Source license. However, the Licensed Work will eventually be made available
under an Open Source License, as stated in this License.
License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
“Business Source License” is a trademark of MariaDB Corporation Ab.
-----------------------------------------------------------------------------
Business Source License 1.1
Terms
The Licensor hereby grants you the right to copy, modify, create derivative
works, redistribute, and make non-production use of the Licensed Work. The
Licensor may make an Additional Use Grant, above, permitting limited
production use.
Effective on the Change Date, or the fourth anniversary of the first publicly
available distribution of a specific version of the Licensed Work under this
License, whichever comes first, the Licensor hereby grants you rights under
the terms of the Change License, and the rights granted in the paragraph
above terminate.
If your use of the Licensed Work does not comply with the requirements
currently in effect as described in this License, you must purchase a
commercial license from the Licensor, its affiliated entities, or authorized
resellers, or you must refrain from using the Licensed Work.
All copies of the original and modified Licensed Work, and derivative works
of the Licensed Work, are subject to this License. This License applies
separately for each version of the Licensed Work and the Change Date may vary
for each version of the Licensed Work released by Licensor.
You must conspicuously display this License on each original or modified copy
of the Licensed Work. If you receive the Licensed Work in original or
modified form from a third party, the terms and conditions set forth in this
License apply to your use of that work.
Any use of the Licensed Work in violation of this License will automatically
terminate your rights under this License for the current and all other
versions of the Licensed Work.
This License does not grant you any right in any trademark or logo of
Licensor or its affiliates (provided that you may use a trademark or logo of
Licensor as expressly required by this License).
TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
AN “AS IS” BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
TITLE.
MariaDB hereby grants you permission to use this License’s text to license
your works, and to refer to it using the trademark “Business Source License”,
as long as you comply with the Covenants of Licensor below.
Covenants of Licensor
In consideration of the right to use this License’s text and the “Business
Source License” name and trademark, Licensor covenants to MariaDB, and to all
other recipients of the licensed work to be provided by Licensor:
1. To specify as the Change License the GPL Version 2.0 or any later version,
or a license that is compatible with GPL Version 2.0 or a later version,
where “compatible” means that software provided under the Change License can
be included in a program with software provided under GPL Version 2.0 or a
later version. Licensor may specify additional Change Licenses without
limitation.
2. To either: (a) specify an additional grant of rights to use that does not
impose any additional restriction on the right granted in this License, as
the Additional Use Grant; or (b) insert the text “None”.
3. To specify a Change Date.
4. Not to modify this License in any other way.

17
backend/build.rs Normal file
View File

@@ -0,0 +1,17 @@
use std::fs::File;
use std::io::Write;
use deno_core::{JsRuntime, RuntimeOptions};
fn main() {
println!("cargo:rerun-if-changed=build.rs");
let options = RuntimeOptions {
will_snapshot: true,
..Default::default()
};
let mut runtime = JsRuntime::new(options);
let mut snap = File::create("v8.snap").expect("can create snap file");
snap.write_all(&runtime.snapshot())
.expect("can write content to snap");
}

0
backend/migrations/.gitkeep Normal file
View File

0
backend/migrations/20220123221903_first.down.sql Normal file
View File

575
backend/migrations/20220123221903_first.up.sql Normal file
View File

@@ -0,0 +1,575 @@
-- Add migration script here
create SCHEMA IF NOT exists extensions;
create extension if not exists "uuid-ossp" with schema extensions;
CREATE TABLE workspace (
id VARCHAR(50) PRIMARY KEY,
name VARCHAR(50) NOT NULL,
owner VARCHAR(50) NOT NULL,
domain VARCHAR(30),
deleted BOOLEAN NOT NULL DEFAULT false,
CONSTRAINT proper_id CHECK (id ~ '^\w+(-\w+)*$')
);
INSERT INTO workspace(id, name, owner) VALUES
('starter', 'Starter', 'admin@windmill.dev'),
('demo', 'Demo', 'admin@windmill.dev');
CREATE TABLE script (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
hash BIGINT NOT NULL,
path varchar(255) NOT NULL,
parent_hashes BIGINT[],
summary TEXT NOT NULL,
description TEXT NOT NULL,
content TEXT NOT NULL,
created_by VARCHAR(50) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
archived BOOLEAN NOT NULL DEFAULT false,
schema JSONB,
deleted BOOLEAN NOT NULL DEFAULT false,
is_template boolean DEFAULT false,
PRIMARY KEY (workspace_id, hash),
CONSTRAINT proper_id CHECK (path ~ '^[ug](\/[\w-]+){2,}$')
);
CREATE TABLE flow (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
path varchar(255) NOT NULL,
summary TEXT NOT NULL,
description TEXT NOT NULL,
value JSONB NOT NULL,
edited_by VARCHAR(50) NOT NULL,
edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
archived BOOLEAN NOT NULL DEFAULT false,
schema JSONB,
PRIMARY KEY (workspace_id, path),
CONSTRAINT proper_id CHECK (path ~ '^[ug](\/[\w-]+){2,}$')
);
INSERT INTO script(workspace_id, created_by, content, schema, summary, description, path, hash) VALUES (
'starter',
'system', 'import wmill
import psycopg2
client = wmill.Client()
def main():
# query that returns rows will return them as a list
res1 = query_pg("SELECT * from demo", "g/all/demodb")
# query that does not return rows will return None
res2 = query_pg("UPDATE demo SET value = ''value''", "g/all/demodb")
# one can use RETURNING to still fetch the updated rows
res3 = query_pg("UPDATE demo SET value = ''value'' RETURNING *", "g/all/demodb")
# output expects a dict
return {"res1": res1, "res2": res2, "res3": res3}
def query_pg(query: str, resource: str):
pg_con = client.get_resource(resource)
conn = psycopg2.connect(**pg_con)
cur = conn.cursor()
cur.execute(f"{query};")
if cur.description:
return cur.fetchall()
else:
return None',
'{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {},
"required": [],
"type": "object"
}',
'Query the demodb resource','
An example of how to use resources from scripts. In this example, we will query the demo database demodb that is set up by default on Windmill.',
'u/bot/postgres_example', 43),
(
'starter',
'system',
'import os
def main(name: str = "Nicolas Bourbaki"):
print(f"Hello World and a warm welcome especially to {name}")
print("The env variable at `g/all/pretty_secret`: ", os.environ.get("G_ALL_PRETTY_SECRET"))
return {"len": len(name), "splitted": name.split() }',
'{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"properties": {
"name": {
"description": "",
"type": "string",
"default": "Nicolas Bourbaki"
}
},
"required": [],
"type": "object"
}',
'Hello World', '', 'u/bot/hello_world', 44);
CREATE INDEX index_script_on_path_created_at ON script (path, created_at);
CREATE TYPE JOB_KIND AS ENUM ('script', 'preview', 'flow', 'dependencies');
CREATE TABLE queue (
id UUID PRIMARY KEY,
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
parent_job UUID,
created_by VARCHAR(50) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
started_at TIMESTAMP WITH TIME ZONE,
scheduled_for TIMESTAMP WITH TIME ZONE NOT NULL,
running BOOLEAN NOT NULL DEFAULT FALSE,
script_hash BIGINT,
script_path VARCHAR(255),
args JSONB,
logs TEXT,
raw_code TEXT,
canceled boolean NOT NULL DEFAULT false,
canceled_by VARCHAR(50),
canceled_reason TEXT,
last_ping TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
job_kind JOB_KIND NOT NULL DEFAULT 'script',
env_id INTEGER,
schedule_path VARCHAR(255),
permissioned_as VARCHAR(55) NOT NULL DEFAULT 'g/all',
flow_status JSONB
);
CREATE INDEX index_queue_on_workspace_id ON queue (workspace_id);
CREATE INDEX index_queue_on_scheduled_for ON queue (scheduled_for);
CREATE INDEX index_queue_on_running ON queue (running);
CREATE INDEX index_queue_on_created ON queue (created_at);
CREATE INDEX index_queue_on_script_path ON queue (script_path);
CREATE INDEX index_queue_on_script_hash ON queue (script_hash);
CREATE TABLE completed_job (
id UUID PRIMARY KEY,
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
parent_job UUID,
created_by VARCHAR(50) NOT NULL,
created_at TIMESTAMP WITH TIME ZONE NOT NULL,
duration INT NOT NULL,
success BOOLEAN NOT NULL,
script_hash BIGINT,
script_path VARCHAR(255),
args JSONB,
result JSONB,
logs TEXT,
deleted BOOLEAN NOT NULL DEFAULT false,
raw_code TEXT,
canceled boolean NOT NULL DEFAULT false,
canceled_by VARCHAR(50),
canceled_reason TEXT,
job_kind JOB_KIND NOT NULL DEFAULT 'script',
env_id INTEGER NOT NULL DEFAULT 0,
schedule_path varchar(255),
permissioned_as VARCHAR(55) NOT NULL DEFAULT 'g/all',
flow_status JSONB
);
CREATE INDEX index_completed_on_workspace_id ON completed_job (workspace_id);
CREATE INDEX index_completed_on_created ON completed_job (created_at);
CREATE INDEX index_completed_on_script_path ON completed_job (script_path);
CREATE INDEX index_completed_on_script_hash ON completed_job (script_hash);
CREATE TABLE usr (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
username VARCHAR(50) NOT NULL,
email VARCHAR(50) NOT NULL,
is_admin BOOLEAN NOT NULL DEFAULT false,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
operator BOOLEAN NOT NULL DEFAULT false,
disabled BOOLEAN NOT NULL DEFAULT false,
role VARCHAR(50),
PRIMARY KEY (workspace_id, username),
CONSTRAINT proper_username CHECK (username ~ '^[\w-]+$'),
CONSTRAINT proper_email CHECK (email ~ '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$')
);
CREATE INDEX index_usr_email ON usr (email);
CREATE TYPE LOGIN_TYPE AS ENUM ('password', 'github');
CREATE TABLE password (
email VARCHAR(50) PRIMARY KEY,
password_hash VARCHAR(100),
login_type LOGIN_TYPE NOT NULL,
super_admin BOOLEAN NOT NULL DEFAULT FALSE,
verified BOOLEAN NOT NULL DEFAULT FALSE,
name VARCHAR(30),
company VARCHAR(30)
);
-- CREATE TABLE invite_code (
-- code VARCHAR(20) PRIMARY KEY,
-- seats_left INTEGER NOT NULL DEFAULT 0,
-- seats_given INTEGER NOT NULL DEFAULT 1
-- );
CREATE TABLE workspace_settings (
workspace_id VARCHAR(50) PRIMARY KEY REFERENCES workspace(id),
slack_team_id VARCHAR(50) UNIQUE,
slack_name VARCHAR(50),
slack_command_script VARCHAR(255)
);
INSERT INTO workspace_settings (workspace_id) VALUES
('starter'),
('demo');
CREATE TABLE workspace_invite (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
email VARCHAR(50),
is_admin bool NOT NULL DEFAULT false,
CONSTRAINT proper_email CHECK (email ~ '^(?:[a-z0-9!#$%&''*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&''*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(2(5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])$'),
PRIMARY KEY (workspace_id, email)
);
CREATE TABLE magic_link (
email VARCHAR(50) NOT NULL,
token VARCHAR(100) NOT NULL,
expiration TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT (NOW() + interval '1 day'),
PRIMARY KEY (email, token)
);
CREATE TABLE token (
token VARCHAR(50) PRIMARY KEY,
label VARCHAR(50),
expiration TIMESTAMP WITH TIME ZONE,
workspace_id VARCHAR(50) REFERENCES workspace(id),
owner VARCHAR(55),
email VARCHAR(50),
super_admin BOOLEAN NOT NULL DEFAULT FALSE,
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW(),
last_used_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
);
CREATE INDEX index_magic_link_exp ON magic_link (expiration);
CREATE INDEX index_token_exp ON token (expiration);
INSERT INTO usr(workspace_id, email, username, is_admin, role) VALUES
('starter', 'admin@windmill.dev', 'admin', true, 'Admin'),
('demo', 'admin@windmill.dev', 'admin', true, 'Ruben');
INSERT INTO password(email, verified, password_hash, login_type, super_admin, name, company) VALUES
('admin@windmill.dev', true, '$argon2id$v=19$m=4096,t=3,p=1$z0Kg3qyaS14e+YHeihkJLQ$N69flI6yQ/U98pjAHtbNxbdz2f4PrJEi9Tx1VoYk1as', 'password', true, 'Admin', 'Windmill'),
('ruben@windmill.dev', true, '$argon2id$v=19$m=4096,t=3,p=1$z0Kg3qyaS14e+YHeihkJLQ$N69flI6yQ/U98pjAHtbNxbdz2f4PrJEi9Tx1VoYk1as', 'password', true, 'Ruben', 'Windmill'),
('user@windmill.dev', true, '$argon2id$v=19$m=4096,t=3,p=1$z0Kg3qyaS14e+YHeihkJLQ$N69flI6yQ/U98pjAHtbNxbdz2f4PrJEi9Tx1VoYk1as', 'password', false, 'User', 'Windmill');
INSERT INTO workspace_invite(workspace_id, email, is_admin) VALUES
('demo', 'ruben@windmill.dev', true);
CREATE TABLE variable (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
path VARCHAR(255),
value VARCHAR(4012) NOT NULL,
is_secret BOOLEAN NOT NULL DEFAULT FALSE,
description VARCHAR(255) NOT NULL DEFAULT '',
PRIMARY KEY (workspace_id, path),
CONSTRAINT proper_id CHECK (path ~ '^[ug](\/[\w-]+){2,}$')
);
-- CREATE TABLE oauth(
-- id VARCHAR(150) NOT NULL PRIMARY KEY,
-- owner VARCHAR(50),
-- workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
-- type VARCHAR(50) NOT NULL,
-- refresh_token VARCHAR(255),
-- access_token VARCHAR(255) NOT NULL
-- );
-- CREATE INDEX index_oauth ON oauth (workspace_id, type, owner);
CREATE TYPE ACTION_KIND AS ENUM ('create', 'update', 'delete', 'execute');
CREATE TABLE audit (
workspace_id VARCHAR(50) NOT NULL,
id SERIAL,
timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
username VARCHAR(50) NOT NULL,
operation VARCHAR(50) NOT NULL,
action_kind ACTION_KIND NOT NULL,
resource VARCHAR(255),
parameters JSONB,
PRIMARY KEY (workspace_id, id)
);
CREATE TABLE resource_type (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
name VARCHAR(50),
schema JSONB,
description TEXT,
PRIMARY KEY (workspace_id, name),
CONSTRAINT proper_name CHECK (name ~ '^[\w-]+$')
);
CREATE TABLE resource (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
path VARCHAR(255),
value JSONB,
description TEXT,
resource_type VARCHAR(50) NOT NULL,
PRIMARY KEY (workspace_id, path),
CONSTRAINT proper_id CHECK (path ~ '^[ug](\/[\w-]+){2,}$')
);
INSERT INTO resource_type(workspace_id, name, schema, description) VALUES
('starter', 'postgres', '{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {
"dbname": {
"description": "The database name",
"type": "string"
},
"user": {
"description": "The postgres username",
"type": "string"
},
"password": {
"description": "The postgres users password",
"type": "string"
},
"sslmode": {
"description": "The sslmode",
"type": "string",
"enum": ["disable", "allow", "prefer", "require", "verify-ca", "verify-full"]
},
"host": {
"description": "The instance host",
"type": "string"
},
"port": {
"description": "The instance port",
"type": "integer"
}
},
"required": ["dbname", "user", "password"]
}', 'A postgres database connection resource')
;
INSERT INTO resource(workspace_id, path, value, description, resource_type) VALUES
('starter', 'g/all/demodb', '{"host": "demodb.service.consul", "dbname": "demodb",
"user": "postgres", "password": "demodb", "sslmode": "disable", "port":"6543"}', 'demodb', 'postgres')
;
CREATE TABLE pipenv (
id SERIAL PRIMARY KEY,
timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
created_by VARCHAR(50) NOT NULL,
python_version VARCHAR(20),
dependencies VARCHAR(255)[] NOT NULL DEFAULT array[]::varchar[],
pipfile_lock TEXT,
job_id UUID
);
CREATE TABLE schedule(
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
path varchar(255),
edited_by varchar(255) NOT NULL,
edited_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
schedule VARCHAR(255) NOT NULL,
offset_ INTEGER NOT NULL,
enabled BOOLEAN NOT NULL DEFAULT true,
script_path varchar(255),
script_hash BIGINT,
args JSONB,
PRIMARY KEY (workspace_id, path),
CONSTRAINT proper_id CHECK (path ~ '^[ug](\/[\w-]+){2,}$')
);
CREATE TABLE group_ (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
name VARCHAR(50),
summary TEXT,
PRIMARY KEY (workspace_id, name),
CONSTRAINT proper_name CHECK (name ~ '^[\w-]+$')
);
CREATE TABLE usr_to_group(
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
group_ VARCHAR(50) NOT NULL,
usr VARCHAR(50) NOT NULL DEFAULT 'ruben',
CONSTRAINT fk_group FOREIGN KEY(workspace_id, group_) REFERENCES group_(workspace_id, name),
PRIMARY KEY (workspace_id, usr, group_)
);
INSERT INTO group_ SELECT id, 'all', 'The group that always contains all users of this workspace' FROM workspace;
CREATE TABLE worker_ping(
worker VARCHAR(50) PRIMARY KEY,
worker_instance VARCHAR(50) NOT NULL,
ping_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
started_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
env_id INTEGER NOT NULL DEFAULT -1,
ip VARCHAR(50) NOT NULL DEFAULT 'NO IP',
jobs_executed INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX worker_ping_on_ping_at ON worker_ping (ping_at);
ALTER TABLE audit ENABLE ROW LEVEL SECURITY;
CREATE POLICY audit_log_see_own ON audit FOR SELECT
USING(audit.username = current_setting('session.user') or current_setting('session.is_admin')::boolean);
-- USING(current_setting('session.is_admin')::boolean);
DO
$do$
BEGIN
IF NOT EXISTS (
SELECT FROM pg_catalog.pg_roles
WHERE rolname = 'app') THEN
CREATE ROLE app LOGIN PASSWORD 'changeme';
END IF;
END
$do$;
GRANT SELECT ON audit TO app;
REVOKE ALL
ON ALL TABLES IN SCHEMA public
FROM PUBLIC;
GRANT ALL
ON ALL TABLES IN SCHEMA public
TO admin;
ALTER DEFAULT PRIVILEGES
FOR ROLE admin
IN SCHEMA public
GRANT ALL ON TABLES TO admin;
INSERT INTO usr_to_group
SELECT workspace_id, 'all', username FROM (SELECT workspace_id, username from usr) as usernames
;
DROP POLICY audit_log_see_own on audit;
GRANT ALL ON audit TO app;
CREATE POLICY see_own ON audit FOR ALL
USING (audit.username = current_setting('session.user'));
GRANT ALL ON queue TO app;
ALTER TABLE queue ENABLE ROW LEVEL SECURITY;
CREATE POLICY see_own ON queue FOR ALL
USING (SPLIT_PART(queue.permissioned_as, '/', 1) = 'u' AND SPLIT_PART(queue.permissioned_as, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON queue FOR ALL
USING (SPLIT_PART(queue.permissioned_as, '/', 1) = 'g' AND SPLIT_PART(queue.permissioned_as, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
GRANT ALL ON completed_job TO app;
ALTER TABLE completed_job ENABLE ROW LEVEL SECURITY;
CREATE POLICY see_starter ON completed_job FOR SELECT
USING (completed_job.workspace_id = 'starter');
CREATE POLICY see_own ON completed_job FOR ALL
USING (SPLIT_PART(completed_job.permissioned_as, '/', 1) = 'u' AND SPLIT_PART(completed_job.permissioned_as, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON completed_job FOR ALL
USING (SPLIT_PART(completed_job.permissioned_as, '/', 1) = 'g' AND SPLIT_PART(completed_job.permissioned_as, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
GRANT SELECT ON pipenv to app;
GRANT SELECT (email, username, is_admin, workspace_id) ON usr to app;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to app;
GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA public to admin;
GRANT SELECT, INSERT ON resource_type to app;
GRANT SELECT ON worker_ping to app;
GRANT SELECT ON worker_ping to admin;
CREATE POLICY schedule ON audit FOR INSERT
WITH CHECK (audit.username LIKE 'schedule-%');
DO
$do$
DECLARE
i text;
arr text[] := array['resource', 'script', 'variable', 'schedule', 'flow'];
BEGIN
FOREACH i IN ARRAY arr
LOOP
EXECUTE FORMAT(
$$
GRANT ALL ON %1$I TO app;
ALTER TABLE %1$I ENABLE ROW LEVEL SECURITY;
CREATE POLICY see_starter ON %1$I FOR SELECT
USING (%1$I.workspace_id = 'starter');
CREATE POLICY see_own ON %1$I FOR ALL
USING (SPLIT_PART(%1$I.path, '/', 1) = 'u' AND SPLIT_PART(%1$I.path, '/', 2) = current_setting('session.user'));
CREATE POLICY see_member ON %1$I FOR ALL
USING (SPLIT_PART(%1$I.path, '/', 1) = 'g' AND SPLIT_PART(%1$I.path, '/', 2) = any(regexp_split_to_array(current_setting('session.groups'), ',')::text[]));
ALTER TABLE %1$I
ADD COLUMN extra_perms JSONB NOT NULL DEFAULT '{}';
CREATE INDEX %1$I_extra_perms ON %1$I USING GIN (extra_perms);
CREATE POLICY see_extra_perms_user ON %1$I FOR ALL
USING (extra_perms ? CONCAT('u/', current_setting('session.user')))
WITH CHECK ((extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean);
CREATE POLICY see_extra_perms_groups ON %1$I FOR ALL
USING (extra_perms ?| regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
WITH CHECK (exists(
SELECT key, value FROM jsonb_each_text(extra_perms)
WHERE SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
$$,
i
);
END LOOP;
END
$do$;
GRANT ALL ON group_ TO app;
ALTER TABLE group_
ADD COLUMN extra_perms JSONB NOT NULL DEFAULT '{}';
CREATE INDEX group_extra_perms ON group_ USING GIN (extra_perms);
GRANT ALL ON usr_to_group TO app;
ALTER TABLE usr_to_group ENABLE ROW LEVEL SECURITY;
CREATE POLICY see_extra_perms_user ON usr_to_group FOR ALL
USING (true)
WITH CHECK (EXISTS(SELECT 1 FROM group_ WHERE usr_to_group.group_ = group_.name AND usr_to_group.workspace_id = group_.workspace_id AND (group_.extra_perms ->> CONCAT('u/', current_setting('session.user')))::boolean));
CREATE POLICY see_extra_perms_groups ON usr_to_group FOR ALL
USING (true)
WITH CHECK (exists(
SELECT f.* FROM group_ g, jsonb_each_text(g.extra_perms) f
WHERE usr_to_group.group_ = g.name AND usr_to_group.workspace_id = g.workspace_id AND SPLIT_PART(key, '/', 1) = 'g' AND key = ANY(regexp_split_to_array(current_setting('session.pgroups'), ',')::text[])
AND value::boolean));
DO
$do$
BEGIN
IF NOT EXISTS (
SELECT FROM pg_catalog.pg_roles -- SELECT list can be empty for this
WHERE rolname = 'admin') THEN
CREATE ROLE admin WITH BYPASSRLS LOGIN PASSWORD 'changeme';
END IF;
END
$do$;

6
backend/migrations/20220316135622_raw_flow.down.sql Normal file
View File

@@ -0,0 +1,6 @@
-- Add down migration script here
ALTER TABLE queue DROP COLUMN raw_flow;
ALTER TABLE completed_job DROP COLUMN raw_flow;
ALTER TABLE queue DROP COLUMN is_flow_step NOT NULL DEFAULT false;
ALTER TABLE completed_job DROP COLUMN is_flow_step NOT NULL DEFAULT false;

6
backend/migrations/20220316135622_raw_flow.up.sql Normal file
View File

@@ -0,0 +1,6 @@
-- Add up migration script here
ALTER TABLE queue ADD COLUMN raw_flow JSONB;
ALTER TABLE completed_job ADD COLUMN raw_flow JSONB;
ALTER TABLE queue ADD COLUMN is_flow_step boolean DEFAULT false;
ALTER TABLE completed_job ADD COLUMN is_flow_step boolean DEFAULT false;

4
backend/migrations/20220320122733_schedule_flow.down.sql Normal file
View File

@@ -0,0 +1,4 @@
-- Add down migration script here
ALTER TABLE schedule ADD COLUMN script_hash BIGINT;
ALTER TABLE schedule DROP COLUMN is_flow;

4
backend/migrations/20220320122733_schedule_flow.up.sql Normal file
View File

@@ -0,0 +1,4 @@
-- Add up migration script here
ALTER TABLE schedule DROP COLUMN script_hash;
ALTER TABLE schedule ADD COLUMN is_flow boolean NOT NULL DEFAULT false;
ALTER TABLE schedule ALTER COLUMN script_path SET NOT NULL;;

1
backend/migrations/20220321004844_flow_preview.down.sql Normal file
View File

@@ -0,0 +1 @@
-- Add down migration script here

2
backend/migrations/20220321004844_flow_preview.up.sql Normal file
View File

@@ -0,0 +1,2 @@
-- Add up migration script here
ALTER TYPE JOB_KIND ADD VALUE 'flowpreview';

20
backend/migrations/20220406141754_custom_env.down.sql Normal file
View File

@@ -0,0 +1,20 @@
-- Add down migration script here
CREATE TABLE pipenv (
id SERIAL PRIMARY KEY,
timestamp TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(),
created_by VARCHAR(50) NOT NULL,
python_version VARCHAR(20),
dependencies VARCHAR(255)[] NOT NULL DEFAULT array[]::varchar[],
pipfile_lock TEXT,
job_id UUID
);
ALTER TABLE script
DROP COLUMN lock;
ALTER TABLE script
DROP COLUMN lock_error_logs;
ALTER TABLE worker_ping
ADD COLUMN env_id INTEGER NOT NULL DEFAULT -1;

11
backend/migrations/20220406141754_custom_env.up.sql Normal file
View File

@@ -0,0 +1,11 @@
-- Add up migration script here
DROP TABLE pipenv;
ALTER TABLE script
ADD COLUMN lock TEXT;
ALTER TABLE script
ADD COLUMN lock_error_logs TEXT;
ALTER TABLE worker_ping
DROP COLUMN env_id;

6
backend/migrations/20220421061414_jsonb.down.sql Normal file
View File

@@ -0,0 +1,6 @@
-- Add down migration script here
ALTER TABLE script
ALTER COLUMN schema TYPE jsonb;
ALTER TABLE flow
ALTER COLUMN schema TYPE jsonb;

6
backend/migrations/20220421061414_jsonb.up.sql Normal file
View File

@@ -0,0 +1,6 @@
-- Add up migration script here
ALTER TABLE script
ALTER COLUMN schema TYPE json;
ALTER TABLE flow
ALTER COLUMN schema TYPE json;

3
backend/migrations/20220428085013_private_key.down.sql Normal file
View File

@@ -0,0 +1,3 @@
-- Add down migration script here
DROP TABLE workspace_key;
DROP TYPE WORKSPACE_KEY_KIND;

16
backend/migrations/20220428085013_private_key.up.sql Normal file
View File

@@ -0,0 +1,16 @@
-- Add up migration script here
CREATE TYPE WORKSPACE_KEY_KIND AS ENUM ('cloud');
CREATE TABLE workspace_key (
workspace_id VARCHAR(50) NOT NULL REFERENCES workspace(id),
kind WORKSPACE_KEY_KIND NOT NULL,
key VARCHAR(255) NOT NULL DEFAULT 'changeme',
PRIMARY KEY (workspace_id, kind)
);
GRANT SELECT ON workspace_key TO app;
GRANT SELECT ON workspace_key TO admin;
INSERT INTO workspace_key SELECT id as workspace_id, 'cloud' as kind, 'changeme' as key FROM workspace;

2
backend/migrations/20220503085923_slack_resource.down.sql Normal file
View File

@@ -0,0 +1,2 @@
-- Add down migration script here
DELETE FROM resource_type WHERE name = 'slack' AND workspace_id = 'starter';

14
backend/migrations/20220503085923_slack_resource.up.sql Normal file
View File

@@ -0,0 +1,14 @@
-- Add up migration script here
INSERT INTO resource_type(workspace_id, name, schema, description) VALUES
('starter', 'slack', '{
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [],
"properties": {
"token": {
"type": "string",
"description": "The slack token"
}
}
}', 'A slack token to interact with a specific workspace. Can be obtained from the OAuth integration in the workspace settings.')
;

3406
backend/openapi.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

7
backend/rustfmt.toml Normal file
View File

@@ -0,0 +1,7 @@
imports_granularity = "Crate"
max_width = 100
use_small_heuristics = "Default"
indent_style = "Block"
fn_single_line = false
force_multiline_blocks = true
format_strings = true

2972
backend/sqlx-data.json Normal file
View File

File diff suppressed because it is too large Load Diff

159
backend/src/audit.rs Normal file
View File

@@ -0,0 +1,159 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use sql_builder::prelude::*;
use std::collections::HashMap;
use crate::{
db::UserDB,
error::{Error, JsonResult, Result},
users::Authed,
utils::Pagination,
};
use axum::{
extract::{Extension, Path, Query},
routing::get,
Json, Router,
};
use serde::{Deserialize, Serialize};
use sql_builder::SqlBuilder;
use sqlx::{FromRow, Postgres, Transaction};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_audit))
.route("/get/:id", get(get_audit))
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug)]
#[sqlx(type_name = "ACTION_KIND", rename_all = "lowercase")]
pub enum ActionKind {
Create,
Update,
Delete,
Execute,
}
#[derive(FromRow, Serialize, Deserialize)]
pub struct AuditLog {
pub workspace_id: String,
pub id: i32,
pub timestamp: chrono::DateTime<chrono::Utc>,
pub username: String,
pub operation: String,
pub action_kind: ActionKind,
pub resource: Option<String>,
pub parameters: Option<serde_json::Value>,
}
pub async fn audit_log<'c>(
db: &mut Transaction<'c, Postgres>,
username: &str,
operation: &str,
action_kind: ActionKind,
w_id: &str,
resource: Option<&str>,
parameters: Option<HashMap<&str, &str>>,
) -> Result<()> {
let p_json: serde_json::Value = serde_json::to_value(&parameters).unwrap();
tracing::info!(
username = username,
kind = "audit",
operation = operation,
workspace = w_id,
action_kind = ?action_kind,
resource = resource,
parameters = %p_json
);
sqlx::query(
"INSERT INTO audit
(workspace_id, username, operation, action_kind, resource, parameters)
VALUES ($1, $2, $3, $4, $5, $6)",
)
.bind(w_id)
.bind(username)
.bind(operation)
.bind(action_kind)
.bind(resource)
.bind(p_json)
.execute(db)
.await?;
Ok(())
}
#[derive(Deserialize)]
pub struct ListAuditLogQuery {
pub username: Option<String>,
pub operation: Option<String>,
pub action_kind: Option<String>,
pub resource: Option<String>,
pub before: Option<chrono::DateTime<chrono::Utc>>,
pub after: Option<chrono::DateTime<chrono::Utc>>,
}
async fn list_audit(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(lq): Query<ListAuditLogQuery>,
) -> JsonResult<Vec<AuditLog>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let mut sqlb = SqlBuilder::select_from("audit")
.field("*")
.order_by("id", true)
.and_where_eq("workspace_id", "?".bind(&w_id))
.offset(offset)
.limit(per_page)
.clone();
if let Some(u) = &lq.username {
sqlb.and_where_eq("username", "?".bind(u));
}
if let Some(o) = &lq.operation {
sqlb.and_where_eq("operation", "?".bind(o));
}
if let Some(ak) = &lq.action_kind {
sqlb.and_where_eq("action_kind", "?".bind(ak));
}
if let Some(r) = &lq.resource {
sqlb.and_where_eq("resource", "?".bind(r));
}
if let Some(b) = &lq.before {
sqlb.and_where_le("timestamp", format!("to_timestamp({})", b.timestamp()));
}
if let Some(a) = &lq.after {
sqlb.and_where_gt("timestamp", format!("to_timestamp({})", a.timestamp()));
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as::<_, AuditLog>(&sql)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
}
async fn get_audit(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(id): Path<i32>,
) -> JsonResult<AuditLog> {
let mut tx = user_db.begin(&authed).await?;
let audit_o = sqlx::query_as::<_, AuditLog>("SELECT * FROM audit WHERE id = $1")
.bind(id)
.fetch_optional(&mut tx)
.await?;
tx.commit().await?;
let audit = crate::utils::not_found_if_none(audit_o, "AuditLog", &id.to_string())?;
Ok(Json(audit))
}

44
backend/src/client.rs Normal file
View File

@@ -0,0 +1,44 @@
use crate::{error::Error, variables::ListableVariable};
pub async fn get_variable(
workspace: &str,
path: &str,
token: &str,
base_url: &str,
) -> Result<String, anyhow::Error> {
let client = reqwest::Client::new();
let res = client
.get(format!("{base_url}/api/w/{workspace}/variables/get/{path}"))
.bearer_auth(token)
.send()
.await?;
if res.status().is_success() {
let value = res
.json::<ListableVariable>()
.await?
.value
.unwrap_or_else(|| "".to_string());
Ok(value)
} else {
Err(Error::NotFound(format!("Variable not found at {path}")))?
}
}
pub async fn get_resource(
workspace: &str,
path: &str,
token: &str,
base_url: &str,
) -> Result<Option<serde_json::Value>, anyhow::Error> {
let client = reqwest::Client::new();
let result = client
.get(format!(
"{base_url}/api/w/{workspace}/resources/get_value/{path}"
))
.bearer_auth(token)
.send()
.await?
.json::<Option<serde_json::Value>>()
.await?;
Ok(result)
}

93
backend/src/db.rs Normal file
View File

@@ -0,0 +1,93 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{error::Error, users::Authed};
use sqlx::{postgres::PgPoolOptions, Pool, Postgres, Transaction};
use std::time::Duration;
pub type DB = Pool<Postgres>;
pub async fn connect(database_url: &str) -> Result<DB, Error> {
PgPoolOptions::new()
.max_connections(100)
.max_lifetime(Duration::from_secs(30 * 60)) // 30 mins
.connect(database_url)
.await
.map_err(|err| Error::ConnectingToDatabase(err.to_string()))
}
pub async fn migrate(db: &DB) -> Result<(), Error> {
match sqlx::migrate!("./migrations").run(db).await {
Ok(_) => Ok(()),
Err(err) => Err(err),
}?;
Ok(())
}
pub async fn setup_app_user(db: &DB, password: &str) -> Result<(), Error> {
let mut tx = db.begin().await?;
sqlx::query(&format!("ALTER USER app WITH PASSWORD '{}'", password))
.execute(&mut tx)
.await?;
sqlx::query(&format!("ALTER USER admin WITH PASSWORD '{}'", password))
.execute(&mut tx)
.await?;
tx.commit().await?;
Ok(())
}
#[derive(Clone)]
pub struct UserDB {
db: DB,
}
impl UserDB {
pub fn new(db: DB) -> Self {
Self { db }
}
pub async fn begin(
self,
authed: &Authed,
) -> Result<Transaction<'static, Postgres>, sqlx::Error> {
let mut tx = self.db.begin().await?;
let user = if authed.is_admin { "admin" } else { "app" };
sqlx::query(&format!("SET LOCAL SESSION AUTHORIZATION {}", user))
.execute(&mut tx)
.await?;
sqlx::query!(
"SELECT set_config('session.user', $1, true)",
authed.username
)
.fetch_optional(&mut tx)
.await?;
sqlx::query!(
"SELECT set_config('session.groups', $1, true)",
&authed.groups.join(",")
)
.fetch_optional(&mut tx)
.await?;
sqlx::query!(
"SELECT set_config('session.pgroups', $1, true)",
&authed
.groups
.iter()
.map(|x| format!("g/{}", x))
.collect::<Vec<_>>()
.join(",")
)
.fetch_optional(&mut tx)
.await?;
Ok(tx)
}
}

31
backend/src/email.rs Normal file
View File

@@ -0,0 +1,31 @@
use lettre::{
transport::smtp::authentication::Credentials, AsyncSmtpTransport, AsyncTransport, Message,
Tokio1Executor,
};
use crate::error::{Error, Result};
pub struct EmailSender {
pub from: String,
pub server: String,
pub password: String,
}
impl EmailSender {
pub async fn send_email(&self, email: Message) -> Result<()> {
let creds = Credentials::new(self.from.to_string(), self.password.to_string());
// Open a remote connection to gmail
let mailer: AsyncSmtpTransport<Tokio1Executor> =
AsyncSmtpTransport::<Tokio1Executor>::relay(&self.server)
.unwrap()
.credentials(creds)
.build();
mailer
.send(email)
.await
.map_err(|x| Error::InternalErr(format!("Impossible to send email {x}")))?;
Ok(())
}
}

67
backend/src/error.rs Normal file
View File

@@ -0,0 +1,67 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{
body::{self, BoxBody},
response::IntoResponse,
Json,
};
use hyper::{Response, StatusCode};
use sqlx::migrate::MigrateError;
use thiserror::Error;
use tokio::io;
pub type Result<T> = std::result::Result<T, Error>;
pub type JsonResult<T> = std::result::Result<Json<T>, Error>;
#[derive(Debug, Error)]
pub enum Error {
#[error("Uuid Error {0}")]
UuidErr(#[from] uuid::Error),
#[error("Bad config: {0}")]
BadConfig(String),
#[error("Connecting to database: {0}")]
ConnectingToDatabase(String),
#[error("Not found: {0}")]
NotFound(String),
#[error("Not authorized: {0}")]
NotAuthorized(String),
#[error("{0}")]
ExecutionErr(String),
#[error("IO error: {0}")]
IoErr(#[from] io::Error),
#[error("Sql error: {0}")]
SqlErr(#[from] sqlx::Error),
#[error("Bad request: {0}")]
BadRequest(String),
#[error("Internal: {0}")]
InternalErr(String),
#[error("Hexadecimal decoding error: {0}")]
HexErr(#[from] hex::FromHexError),
#[error("Migrating database: {0}")]
DatabaseMigration(#[from] MigrateError),
#[error("{0}")]
Anyhow(#[from] anyhow::Error),
}
pub fn to_anyhow<T: 'static + std::error::Error + Send + Sync>(e: T) -> anyhow::Error {
From::from(e)
}
impl IntoResponse for Error {
fn into_response(self) -> Response<BoxBody> {
let e = &self;
let body = body::boxed(body::Full::from(e.to_string()));
let status = match self {
Self::NotFound(_) => StatusCode::NOT_FOUND,
Self::NotAuthorized(_) => StatusCode::UNAUTHORIZED,
Self::SqlErr(_) | Self::BadRequest(_) => StatusCode::BAD_REQUEST,
_ => StatusCode::INTERNAL_SERVER_ERROR,
};
Response::builder().status(status).body(body).unwrap()
}
}

326
backend/src/flow.rs Normal file
View File

@@ -0,0 +1,326 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::HashMap;
use sql_builder::prelude::*;
use axum::{
extract::{Extension, Path, Query},
routing::{get, post},
Json, Router,
};
use serde::{Deserialize, Serialize};
use sql_builder::SqlBuilder;
use sqlx::FromRow;
use crate::{
audit::{audit_log, ActionKind},
db::UserDB,
error::{Error, JsonResult, Result},
scripts::Schema,
users::Authed,
utils::{Pagination, StripPath},
};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_flows))
.route("/create", post(create_flow))
.route("/update/*path", post(update_flow))
.route("/archive/*path", post(archive_flow_by_path))
.route("/get/*path", get(get_flow_by_path))
}
#[derive(FromRow, Serialize)]
pub struct Flow {
pub workspace_id: String,
pub path: String,
pub summary: String,
pub description: String,
pub value: serde_json::Value,
pub edited_by: String,
pub edited_at: chrono::DateTime<chrono::Utc>,
pub archived: bool,
pub schema: Option<Schema>,
pub extra_perms: serde_json::Value,
}
#[derive(FromRow, Deserialize)]
pub struct NewFlow {
pub path: String,
pub summary: String,
pub description: String,
pub value: serde_json::Value,
pub schema: Option<Schema>,
}
#[derive(Deserialize, Serialize)]
pub struct FlowValue {
pub modules: Vec<FlowModule>,
pub failure_module: Option<FlowModule>,
}
#[derive(Deserialize, Serialize)]
pub struct FlowModule {
pub input_transform: HashMap<String, InputTransform>,
pub value: FlowModuleValue,
}
#[derive(Deserialize, Serialize)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum InputTransform {
Static { value: serde_json::Value },
Javascript { expr: String },
Resource { path: String },
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(
tag = "type",
rename_all(serialize = "lowercase", deserialize = "lowercase")
)]
pub enum FlowModuleValue {
Script { path: String },
Flow { path: String },
}
#[derive(Deserialize)]
pub struct ListFlowQuery {
pub path_start: Option<String>,
pub path_exact: Option<String>,
pub edited_by: Option<String>,
pub show_archived: Option<bool>,
pub order_by: Option<String>,
pub order_desc: Option<bool>,
}
async fn list_flows(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(lq): Query<ListFlowQuery>,
) -> JsonResult<Vec<Flow>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let mut sqlb = SqlBuilder::select_from("flow as o")
.fields(&[
"workspace_id",
"path",
"summary",
"description",
"'{}'::jsonb as value",
"edited_by",
"edited_at",
"archived",
"schema",
"extra_perms",
])
.order_by("edited_at", lq.order_desc.unwrap_or(true))
.and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
.offset(offset)
.limit(per_page)
.clone();
if !lq.show_archived.unwrap_or(false) {
sqlb.and_where_eq("archived", false);
}
if let Some(ps) = &lq.path_start {
sqlb.and_where_like_left("path", "?".bind(ps));
}
if let Some(p) = &lq.path_exact {
sqlb.and_where_eq("path", "?".bind(p));
}
if let Some(cb) = &lq.edited_by {
sqlb.and_where_eq("edited_by", "?".bind(cb));
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as::<_, Flow>(&sql).fetch_all(&mut tx).await?;
tx.commit().await?;
Ok(Json(rows))
}
async fn create_flow(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(nf): Json<NewFlow>,
) -> Result<String> {
// cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"INSERT INTO flow (workspace_id, path, summary, description, value, edited_by, edited_at, schema) VALUES ($1, $2, $3, $4, $5, $6, $7, $8::text::json)",
w_id,
nf.path,
nf.summary,
nf.description,
nf.value,
&authed.username,
&chrono::Utc::now(),
nf.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"flows.create",
ActionKind::Create,
&w_id,
Some(&nf.path.to_string()),
Some(
[Some(("flow", nf.path.as_str()))]
.into_iter()
.flatten()
.collect(),
),
)
.await?;
tx.commit().await?;
Ok(nf.path.to_string())
}
async fn update_flow(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, flow_path)): Path<(String, StripPath)>,
Json(nf): Json<NewFlow>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
let flow_path = flow_path.to_path();
let schema = nf.schema.map(|x| x.0);
let flow = sqlx::query_scalar!(
"UPDATE flow SET path = $1, summary = $2, description = $3, value = $4, edited_by = $5, edited_at = $6, schema = $7 WHERE path = $8 AND workspace_id = $9 RETURNING path",
nf.path,
nf.summary,
nf.description,
nf.value,
&authed.username,
&chrono::Utc::now(),
schema,
flow_path,
w_id,
)
.fetch_optional(&mut tx)
.await?;
crate::utils::not_found_if_none(flow, "Flow", flow_path)?;
audit_log(
&mut tx,
&authed.username,
"flows.update",
ActionKind::Create,
&w_id,
Some(&nf.path.to_string()),
Some(
[Some(("flow", nf.path.as_str()))]
.into_iter()
.flatten()
.collect(),
),
)
.await?;
tx.commit().await?;
Ok(nf.path.to_string())
}
async fn get_flow_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Flow> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let flow_o = sqlx::query_as::<_, Flow>(
"SELECT * FROM flow WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
)
.bind(path)
.bind(w_id)
.fetch_optional(&mut tx)
.await?;
tx.commit().await?;
let flow = crate::utils::not_found_if_none(flow_o, "Flow", path)?;
Ok(Json(flow))
}
async fn archive_flow_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"UPDATE flow SET archived = true WHERE path = $1 AND workspace_id = $2",
path,
&w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"flows.archive",
ActionKind::Delete,
&w_id,
Some(path),
Some([("workspace", w_id.as_str())].into()),
)
.await?;
tx.commit().await?;
Ok(format!("Flow {path} archived"))
}
#[cfg(test)]
mod tests {
// Note this useful idiom: importing names from outer (for mod tests) scope.
use super::*;
#[test]
fn test_serialize() -> anyhow::Result<()> {
let mut hm = HashMap::new();
hm.insert(
"test".to_owned(),
InputTransform::Static {
value: serde_json::json!("test2"),
},
);
let fv = FlowValue {
modules: vec![FlowModule {
input_transform: hm,
value: FlowModuleValue::Script {
path: "test".to_string(),
},
}],
failure_module: Some(FlowModule {
input_transform: HashMap::new(),
value: FlowModuleValue::Flow {
path: "test".to_string(),
},
}),
};
println!("{}", serde_json::json!(fv).to_string());
Ok(())
}
}

116
backend/src/granular_acls.rs Normal file
View File

@@ -0,0 +1,116 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
db::UserDB,
error::{Error, JsonResult, Result},
users::Authed,
utils::StripPath,
};
use axum::{
extract::{Extension, Path},
routing::{get, post},
Json, Router,
};
use serde::{Deserialize, Serialize};
pub fn workspaced_service() -> Router {
Router::new()
.route("/get/*path", get(get_granular_acls))
.route("/add/*path", post(add_granular_acl))
.route("/remove/*path", post(remove_granular_acl))
}
#[derive(Serialize, Deserialize)]
pub struct GranularAcl {
pub owner: String,
pub write: Option<bool>,
}
async fn add_granular_acl(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(GranularAcl { owner, write }): Json<GranularAcl>,
) -> Result<String> {
let path = path.to_path();
let (kind, path) = path
.split_once('/')
.ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let identifier = if kind == "group_" { "name" } else { "path" };
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
"UPDATE {kind} SET extra_perms = jsonb_set(extra_perms, '{{\"{owner}\"}}', to_jsonb($1), true) WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
))
.bind(write.unwrap_or(false))
.bind(path)
.bind(&w_id)
.fetch_optional(&mut tx)
.await?;
let _ = crate::utils::not_found_if_none(obj_o, &kind, &path)?;
tx.commit().await?;
Ok("Successfully modified granular acl".to_string())
}
async fn remove_granular_acl(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(GranularAcl { owner, write: _ }): Json<GranularAcl>,
) -> Result<String> {
let path = path.to_path();
let (kind, path) = path
.split_once('/')
.ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let identifier = if kind == "group_" { "name" } else { "path" };
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
"UPDATE {kind} SET extra_perms = extra_perms - $1 WHERE {identifier} = $2 AND workspace_id = $3 RETURNING extra_perms"
))
.bind(owner)
.bind(path)
.bind(w_id)
.fetch_optional(&mut tx)
.await?;
let _ = crate::utils::not_found_if_none(obj_o, &kind, &path)?;
tx.commit().await?;
Ok("Successfully removed granular acl".to_string())
}
async fn get_granular_acls(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<serde_json::Value> {
let path = path.to_path();
let (kind, path) = path
.split_once('/')
.ok_or_else(|| Error::BadRequest("Invalid path or kind".to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let identifier = if kind == "group_" { "name" } else { "path" };
let obj_o = sqlx::query_scalar::<_, serde_json::Value>(&format!(
"SELECT extra_perms from {kind} WHERE {identifier} = $1 AND workspace_id = $2"
))
.bind(path)
.bind(w_id)
.fetch_optional(&mut tx)
.await?;
let obj = crate::utils::not_found_if_none(obj_o, &kind, &path)?;
tx.commit().await?;
Ok(Json(obj))
}

330
backend/src/groups.rs Normal file
View File

@@ -0,0 +1,330 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
users::{owner_to_token_owner, Authed},
utils::Pagination,
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use serde::{Deserialize, Serialize};
use sqlx::{FromRow, Postgres, Transaction};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_groups))
.route("/listnames", get(list_group_names))
.route("/create", post(create_group))
.route("/get/:name", get(get_group))
.route("/update/:name", post(update_group))
.route("/delete/:name", delete(delete_group))
.route("/adduser/:name", post(add_user))
.route("/removeuser/:name", post(remove_user))
}
#[derive(FromRow, Serialize, Deserialize)]
pub struct Group {
pub workspace_id: String,
pub name: String,
pub summary: Option<String>,
pub extra_perms: serde_json::Value,
}
#[derive(Deserialize)]
pub struct NewGroup {
pub name: String,
pub summary: Option<String>,
}
#[derive(Serialize)]
pub struct GroupInfo {
pub workspace_id: String,
pub name: String,
pub summary: Option<String>,
pub members: Vec<String>,
pub extra_perms: serde_json::Value,
}
#[derive(Deserialize)]
pub struct EditGroup {
pub summary: Option<String>,
}
#[derive(Deserialize)]
pub struct Username {
pub username: String,
}
async fn list_groups(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
) -> JsonResult<Vec<Group>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let rows = sqlx::query_as!(
Group,
"SELECT * FROM group_ WHERE workspace_id = $1 ORDER BY name desc LIMIT $2 OFFSET $3",
w_id,
per_page as i64,
offset as i64
)
.fetch_all(&db)
.await?;
Ok(Json(rows))
}
async fn list_group_names(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<String>> {
let rows = sqlx::query_scalar!(
"SELECT name FROM group_ WHERE workspace_id = $1 ORDER BY name desc",
w_id
)
.fetch_all(&db)
.await?;
Ok(Json(rows))
}
async fn create_group(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(ng): Json<NewGroup>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query_as!(
Group,
"INSERT INTO group_ (workspace_id, name, summary, extra_perms) VALUES ($1, $2, $3, $4)",
w_id,
ng.name,
ng.summary,
serde_json::json!({owner_to_token_owner(&authed.username, false): true})
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"group.create",
ActionKind::Create,
&w_id,
Some(&ng.name.to_string()),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Created group {}", ng.name))
}
pub async fn get_group_opt<'c>(
db: &mut Transaction<'c, Postgres>,
w_id: &str,
name: &str,
) -> Result<Option<Group>> {
let group_opt = sqlx::query_as!(
Group,
"SELECT * FROM group_ WHERE name = $1 AND workspace_id = $2",
name,
w_id
)
.fetch_optional(db)
.await?;
Ok(group_opt)
}
async fn get_group(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
) -> JsonResult<GroupInfo> {
let mut tx = user_db.begin(&authed).await?;
let group = crate::utils::not_found_if_none(
get_group_opt(&mut tx, &w_id, &name).await?,
"Group",
&name,
)?;
let members = sqlx::query_scalar!(
"SELECT usr.username
FROM usr_to_group LEFT JOIN usr ON usr_to_group.usr = usr.username
WHERE group_ = $1 AND usr.workspace_id = $2 AND usr_to_group.workspace_id = $2",
name,
w_id
)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(GroupInfo {
workspace_id: group.workspace_id,
name: group.name,
summary: group.summary,
members,
extra_perms: group.extra_perms,
}))
}
async fn delete_group(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
sqlx::query!(
"DELETE FROM usr_to_group WHERE group_ = $1 AND workspace_id = $2",
name,
w_id
)
.execute(&mut tx)
.await?;
sqlx::query!(
"DELETE FROM group_ WHERE name = $1 AND workspace_id = $2",
name,
w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"group.delete",
ActionKind::Delete,
&w_id,
Some(&name.to_string()),
None,
)
.await?;
tx.commit().await?;
Ok(format!("delete group at name {}", name))
}
async fn update_group(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
Json(eg): Json<EditGroup>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
sqlx::query_as!(
Group,
"UPDATE group_ SET summary = $1 WHERE name = $2 AND workspace_id = $3",
eg.summary,
&name,
&w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"group.edit",
ActionKind::Update,
&w_id,
Some(&name.to_string()),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Edited group {}", name))
}
async fn add_user(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
Json(Username {
username: user_username,
}): Json<Username>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
sqlx::query_as!(
Group,
"INSERT INTO usr_to_group (workspace_id, usr, group_) VALUES ($1, $2, $3)",
&w_id,
user_username,
name,
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"group.adduser",
ActionKind::Update,
&w_id,
Some(&name.to_string()),
Some([("user", user_username.as_str())].into()),
)
.await?;
tx.commit().await?;
Ok(format!("Added {} to group {}", user_username, name))
}
async fn remove_user(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
Json(Username {
username: user_username,
}): Json<Username>,
) -> Result<String> {
let mut tx = user_db.begin(&authed).await?;
crate::utils::not_found_if_none(get_group_opt(&mut tx, &w_id, &name).await?, "Group", &name)?;
if &name == "all" {
return Err(Error::BadRequest(format!("Cannot delete users from all")));
}
sqlx::query_as!(
Group,
"DELETE FROM usr_to_group WHERE usr = $1 AND group_ = $2 AND workspace_id = $3",
user_username,
name,
&w_id,
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"group.removeuser",
ActionKind::Update,
&w_id,
Some(&name.to_string()),
Some([("user", user_username.as_str())].into()),
)
.await?;
tx.commit().await?;
Ok(format!("Removed {} to group {}", user_username, name))
}

1525
backend/src/jobs.rs Normal file
View File

File diff suppressed because it is too large Load Diff

300
backend/src/js_eval.rs Normal file
View File

@@ -0,0 +1,300 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::cell::RefCell;
use std::rc::Rc;
use deno_core::serde_v8;
use deno_core::v8;
use deno_core::v8::IsolateHandle;
use deno_core::JsRuntime;
use deno_core::OpState;
use deno_core::RuntimeOptions;
use deno_core::Snapshot;
use deno_core::ZeroCopyBuf;
use itertools::Itertools;
use regex::Regex;
use serde_json::Value;
use tokio::sync::oneshot;
use tokio::time::timeout;
use crate::client;
use crate::error::Error;
pub async fn eval_timeout(
expr: String,
env: Vec<(String, serde_json::Value)>,
workspace: &str,
token: &str,
steps: Vec<String>,
) -> anyhow::Result<serde_json::Value> {
let expr2 = expr.clone();
let (sender, mut receiver) = oneshot::channel::<IsolateHandle>();
let (workspace, token) = (workspace.to_string().clone(), token.to_string().clone());
timeout(
std::time::Duration::from_millis(2000),
tokio::task::spawn_blocking(move || {
let buffer = include_bytes!("../v8.snap");
// Use our snapshot to provision our new runtime
let options = RuntimeOptions {
startup_snapshot: Some(Snapshot::Static(buffer)),
..Default::default()
};
let mut js_runtime = JsRuntime::new(options);
js_runtime.register_op("variable", deno_core::op_async(op_variable));
js_runtime.register_op("resource", deno_core::op_async(op_resource));
if !steps.is_empty() {
js_runtime.register_op("result", deno_core::op_async(op_get_result));
}
js_runtime.sync_ops_cache();
sender
.send(js_runtime.v8_isolate().thread_safe_handle())
.map_err(|_| Error::ExecutionErr("impossible to send v8 isolate".to_string()))?;
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()?;
let re = Regex::new(r"import (.*)\n").unwrap();
let expr = re.replace_all(&expr, "").to_string();
// pretty frail but this it to make the expr more user friendly and not require the user to write await
let expr = ["variable", "step"]
.into_iter()
.fold(expr, replace_with_await);
let r =
runtime.block_on(eval(&mut js_runtime, &expr, env, &workspace, &token, steps))?;
Ok(r) as anyhow::Result<Value>
}),
)
.await
.map_err(|_| {
if let Ok(isolate) = receiver.try_recv() {
isolate.terminate_execution();
};
Error::ExecutionErr(format!(
"The expression of evaluation `{expr2}` took too long to execute (>2000ms)"
))
})??
}
fn replace_with_await(expr: String, fn_name: &str) -> String {
let sep = format!("{}(", fn_name);
let mut split = expr.split(&sep);
let mut s = split.next().unwrap_or_else(|| "").to_string();
for x in split {
s.push_str(&format!("(await {}({}", fn_name, add_closing_bracket(x)))
}
s
}
fn add_closing_bracket(s: &str) -> String {
let mut s = s.to_string();
let mut level = 1;
let mut idx = 0;
for c in s.chars() {
match c {
'(' => level += 1,
')' => level -= 1,
_ => (),
};
if level == 0 {
break;
}
idx += 1;
}
s.insert_str(idx, ")");
s
}
const SPLIT_PAT: &str = ";\n";
async fn eval(
context: &mut JsRuntime,
expr: &str,
env: Vec<(String, serde_json::Value)>,
workspace: &str,
token: &str,
steps: Vec<String>,
) -> anyhow::Result<serde_json::Value> {
let expr = expr.trim();
let expr = format!(
"{}\nreturn {};",
expr.split(SPLIT_PAT)
.take(expr.split(SPLIT_PAT).count() - 1)
.join("\n"),
expr.split(SPLIT_PAT).last().unwrap_or_else(|| "")
);
let steps_code = if !steps.is_empty() {
format!(
r#"
let steps = [{}];
async function step(n) {{
if (n == 0) {{
return flow_input;
}}
if (n == -1) {{
return previous_result;
}}
let token = "{token}";
if (n < 0) {{
let steps_length = steps.length;
n = n % steps.length + steps.length;
}}
let id = steps[n];
return await Deno.core.opAsync("result", [workspace, id, token, base_url]);
}}"#,
steps.into_iter().map(|x| format!("\"{x}\"")).join(",")
)
} else {
"".to_string()
};
let code = format!(
r#"
let workspace = "{workspace}";
let base_url = "{}";
async function variable(path) {{
let token = "{token}";
return await Deno.core.opAsync("variable", [workspace, path, token, base_url]);
}}
async function resource(path) {{
let token = "{token}";
return await Deno.core.opAsync("resource", [workspace, path, token, base_url]);
}}
{}
{steps_code}
(async () => {{
{expr}
}})()
"#,
std::env::var("BASE_INTERNAL_URL")
.unwrap_or_else(|_| "http://missing-base-url".to_string()),
env.into_iter()
.map(|(a, b)| format!(
"let {a} = {};\n",
serde_json::to_string(&b)
.unwrap_or_else(|_| "\"error serializing value\"".to_string())
))
.join(""),
);
let global = context.execute_script("<anon>", &code)?;
let global = context.resolve_value(global).await?;
let scope = &mut context.handle_scope();
let local = v8::Local::new(scope, global);
// Deserialize a `v8` object into a Rust type using `serde_v8`,
// in this case deserialize to a JSON `Value`.
Ok(serde_v8::from_v8::<serde_json::Value>(scope, local)?)
}
// #[warn(dead_code)]
// async fn op_test(
// _state: Rc<RefCell<OpState>>,
// path: String,
// _buf: Option<ZeroCopyBuf>,
// ) -> Result<String, anyhow::Error> {
// tokio::time::sleep(std::time::Duration::from_secs(1)).await;
// Ok(path)
// }
async fn op_variable(
_state: Rc<RefCell<OpState>>,
args: Vec<String>,
_buf: Option<ZeroCopyBuf>,
) -> Result<String, anyhow::Error> {
let workspace = &args[0];
let path = &args[1];
let token = &args[2];
let base_url = &args[3];
client::get_variable(workspace, path, token, &base_url).await
}
async fn op_get_result(
_state: Rc<RefCell<OpState>>,
args: Vec<String>,
_buf: Option<ZeroCopyBuf>,
) -> Result<Option<serde_json::Value>, anyhow::Error> {
let workspace = &args[0];
let id = &args[1];
let token = &args[2];
let base_url = &args[3];
let client = reqwest::Client::new();
let result = client
.get(format!(
"{base_url}/api/w/{workspace}/jobs/completed/get_result/{id}"
))
.bearer_auth(token)
.send()
.await?
.json::<Option<serde_json::Value>>()
.await?;
Ok(result)
}
async fn op_resource(
_state: Rc<RefCell<OpState>>,
args: Vec<String>,
_buf: Option<ZeroCopyBuf>,
) -> Result<Option<serde_json::Value>, anyhow::Error> {
let workspace = &args[0];
let path = &args[1];
let token = &args[2];
let base_url = &args[3];
client::get_resource(workspace, path, token, &base_url).await
}
#[cfg(test)]
mod tests {
use serde_json::json;
// Note this useful idiom: importing names from outer (for mod tests) scope.
use super::*;
#[tokio::test]
async fn test_eval() -> anyhow::Result<()> {
let env = vec![
("params".to_string(), json!({"test": 2})),
("value".to_string(), json!({"test": 2})),
];
let code = "value.test + params.test";
let mut runtime = JsRuntime::new(RuntimeOptions::default());
let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
assert_eq!(res, json!(4));
Ok(())
}
#[tokio::test]
async fn test_eval_multiline() -> anyhow::Result<()> {
let env = vec![];
let code = "let x = 5;
`my ${x}
multiline template`";
let mut runtime = JsRuntime::new(RuntimeOptions::default());
let res = eval(&mut runtime, code, env, "workspace", "token", vec![]).await?;
assert_eq!(res, json!("my 5\nmultiline template"));
Ok(())
}
#[tokio::test]
async fn test_eval_timeout() -> anyhow::Result<()> {
let env = vec![
("params".to_string(), json!({"test": 2})),
("value".to_string(), json!({"test": 2})),
];
let code = r#"variable("test")"#;
let res = eval_timeout(code.to_string(), env, "workspace", "token", vec![]).await?;
assert_eq!(res, json!("test"));
Ok(())
}
}

342
backend/src/lib.rs Normal file
View File

@@ -0,0 +1,342 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use ::oauth2::basic::BasicClient;
use argon2::Argon2;
use axum::{extract::extractor_middleware, handler::Handler, routing::get, Extension, Router};
use db::DB;
use git_version::git_version;
use hyper::Response;
use slack_http_verifier::SlackVerifier;
use std::{collections::HashMap, net::SocketAddr, sync::Arc};
use tokio::sync::Mutex;
use tower::ServiceBuilder;
use tower_cookies::CookieManagerLayer;
use tower_http::trace::{MakeSpan, OnResponse, TraceLayer};
use tracing::{field, Span};
use tracing_subscriber::{filter::filter_fn, prelude::*, EnvFilter};
extern crate magic_crypt;
extern crate dotenv;
mod audit;
mod client;
mod db;
mod email;
mod error;
mod flow;
mod granular_acls;
mod groups;
mod jobs;
mod js_eval;
mod oauth2;
mod parser;
mod resources;
mod schedule;
mod scripts;
mod static_assets;
mod users;
mod utils;
mod variables;
mod worker;
mod worker_ping;
mod workspaces;
use error::Error;
pub use crate::email::EmailSender;
use crate::{db::UserDB, utils::rd_string};
const GIT_VERSION: &str = git_version!(args = ["--tag", "--always"], fallback = "unknown-version");
pub const DEFAULT_NUM_WORKERS: usize = 3;
pub const DEFAULT_TIMEOUT: i32 = 300;
pub const DEFAULT_SLEEP_QUEUE: u64 = 50;
#[derive(Clone)]
struct MyOnResponse {}
impl<B> OnResponse<B> for MyOnResponse {
fn on_response(
self,
response: &Response<B>,
latency: std::time::Duration,
_span: &tracing::Span,
) {
tracing::info!(
latency = %latency.as_millis(),
status = ?response.status(),
"finished processed request")
}
}
#[derive(Clone)]
struct MyMakeSpan {}
impl<B> MakeSpan<B> for MyMakeSpan {
fn make_span(&mut self, request: &hyper::Request<B>) -> Span {
tracing::info_span!(
"request",
method = %request.method(),
uri = %request.uri(),
version = ?request.version(),
username = field::Empty,
)
}
}
pub async fn initialize_tracing() -> anyhow::Result<()> {
//let log_level = if std::env::var("RUST_LOG").map(|x| &x == "debug")
let ts_base = tracing_subscriber::registry()
.with(
EnvFilter::from_default_env()
//.add_directive("windmill".parse()?)
.add_directive("runtime=trace".parse()?)
.add_directive("tokio=trace".parse()?),
)
.with(
tracing_subscriber::fmt::layer()
.json()
.flatten_event(true)
.with_span_list(false)
.with_current_span(true)
.with_filter(filter_fn(|meta| meta.target().starts_with("windmill"))),
);
if std::env::var("TOKIO_CONSOLE")
.map(|x| x == "true")
.unwrap_or(false)
{
let console_layer = console_subscriber::spawn();
ts_base.with(console_layer).init();
} else {
ts_base.init();
}
Ok(())
}
pub async fn migrate_db(db: &DB) -> anyhow::Result<()> {
let app_password = std::env::var("APP_USER_PASSWORD").unwrap_or_else(|_| "changeme".to_owned());
db::migrate(db).await?;
db::setup_app_user(db, &app_password).await?;
Ok(())
}
pub async fn connect_db() -> anyhow::Result<DB> {
let database_url = std::env::var("DATABASE_URL")
.map_err(|_| Error::BadConfig("DATABASE_URL env var is missing".to_string()))?;
Ok(db::connect(&database_url).await?)
}
type BasicClientsMap = HashMap<String, BasicClient>;
pub fn build_oauth_clients(base_url: &str) -> BasicClientsMap {
[(
"github".to_string(),
oauth2::build_gh_client(
&std::env::var("GITHUB_OAUTH_CLIENT_ID").unwrap_or_else(|_| "".to_string()),
&std::env::var("GITHUB_OAUTH_CLIENT_SECRET").unwrap_or_else(|_| "".to_string()),
base_url,
),
)]
.into()
}
#[derive(Clone)]
struct BaseUrl(String);
pub async fn run_server(
db: DB,
addr: SocketAddr,
base_url: &str,
es: EmailSender,
mut rx: tokio::sync::broadcast::Receiver<()>,
) -> anyhow::Result<()> {
let user_db = UserDB::new(db.clone());
let auth_cache = Arc::new(users::AuthCache::new(db.clone()));
let argon2 = Arc::new(Argon2::default());
let email_sender = Arc::new(es);
let basic_clients = Arc::new(build_oauth_clients(base_url));
let slack_verifier = Arc::new(
std::env::var("SLACK_SIGNING_SECRET")
.ok()
.map(|x| SlackVerifier::new(x).unwrap()),
);
let middleware_stack = ServiceBuilder::new()
.layer(
TraceLayer::new_for_http()
.on_response(MyOnResponse {})
.make_span_with(MyMakeSpan {})
.on_request(()),
)
.layer(Extension(db.clone()))
.layer(Extension(user_db))
.layer(Extension(auth_cache.clone()))
.layer(Extension(basic_clients))
.layer(Extension(BaseUrl(base_url.to_string())))
.layer(CookieManagerLayer::new());
// build our application with a route
let app = Router::new()
.nest(
"/api",
Router::new()
.nest(
"/w/:workspace_id",
Router::new()
.nest("/scripts", scripts::workspaced_service())
.nest("/jobs", jobs::workspaced_service())
.nest(
"/users",
users::workspaced_service()
.layer(Extension(argon2.clone()))
.layer(Extension(email_sender)),
)
.nest("/variables", variables::workspaced_service())
.nest("/oauth", oauth2::workspaced_service())
.nest("/resources", resources::workspaced_service())
.nest("/schedules", schedule::workspaced_service())
.nest("/groups", groups::workspaced_service())
.nest("/audit", audit::workspaced_service())
.nest("/acls", granular_acls::workspaced_service())
.nest("/workspaces", workspaces::workspaced_service())
.nest("/flows", flow::workspaced_service()),
)
.nest("/workspaces", workspaces::global_service())
.nest(
"/users",
users::global_service().layer(Extension(argon2.clone())),
)
.nest("/workers", worker_ping::global_service())
.nest("/scripts", scripts::global_service())
.nest("/schedules", schedule::global_service())
.route_layer(extractor_middleware::<users::Authed>())
.route_layer(extractor_middleware::<users::Tokened>())
.nest(
"/auth",
users::make_unauthed_service().layer(Extension(argon2)),
)
.nest(
"/oauth",
oauth2::global_service().layer(Extension(slack_verifier)),
)
.route("/version", get(git_v))
.route("/openapi.yaml", get(openapi)),
)
.fallback(static_assets::static_handler.into_service())
.layer(middleware_stack);
let instance_name = rd_string(5);
tracing::info!(addr = %addr.to_string(), instance = %instance_name, "server started listening");
let server = axum::Server::bind(&addr)
.serve(app.into_make_service())
.with_graceful_shutdown(async {
rx.recv().await.ok();
println!("Graceful shutdown of server");
});
tokio::spawn(async move { auth_cache.monitor().await });
server.await?;
Ok(())
}
pub fn monitor_db(db: &DB, timeout: i32, tx: tokio::sync::broadcast::Sender<()>) {
let db1 = db.clone();
let db2 = db.clone();
let rx1 = tx.subscribe();
let rx2 = tx.subscribe();
tokio::spawn(async move { worker::restart_zombie_jobs_periodically(&db1, timeout, rx1).await });
tokio::spawn(async move { users::delete_expired_items_perdiodically(&db2, rx2).await });
}
pub async fn run_workers(
db: DB,
addr: SocketAddr,
timeout: i32,
num_workers: i32,
sleep_queue: u64,
base_url: String,
tx: tokio::sync::broadcast::Sender<()>,
) -> anyhow::Result<()> {
let instance_name = rd_string(5);
let mutex = Arc::new(Mutex::new(0));
let sources: external_ip::Sources = external_ip::get_http_sources();
let consensus = external_ip::ConsensusBuilder::new()
.add_sources(sources)
.build();
let ip = consensus
.get_consensus()
.await
.map(|x| x.to_string())
.unwrap_or_else(|| "Unretrievable ip".to_string());
let mut handles = Vec::new();
for i in 1..(num_workers + 1) {
let db1 = db.clone();
let instance_name = instance_name.clone();
let worker_name = format!("dt-worker-{}-{}", &instance_name, rd_string(5));
let m1 = mutex.clone();
let ip = ip.clone();
let tx = tx.clone();
let base_url = base_url.clone();
handles.push(tokio::spawn(async move {
tracing::info!(addr = %addr.to_string(), worker = %worker_name, "starting worker");
worker::run_worker(
&db1,
timeout,
&instance_name,
worker_name,
i as u64,
num_workers as u64,
m1,
&ip,
sleep_queue,
&base_url,
tx,
)
.await
}));
}
futures::future::try_join_all(handles).await?;
Ok(())
}
async fn git_v() -> &'static str {
GIT_VERSION
}
async fn openapi() -> &'static str {
include_str!("../openapi.yaml")
}
pub async fn shutdown_signal(tx: tokio::sync::broadcast::Sender<()>) -> anyhow::Result<()> {
use std::io;
use tokio::signal::unix::SignalKind;
async fn terminate() -> io::Result<()> {
tokio::signal::unix::signal(SignalKind::terminate())?
.recv()
.await;
Ok(())
}
tokio::select! {
_ = terminate() => {},
_ = tokio::signal::ctrl_c() => {},
}
println!("signal received, starting graceful shutdown");
let _ = tx.send(());
Ok(())
}

95
backend/src/main.rs Normal file
View File

@@ -0,0 +1,95 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::net::SocketAddr;
use dotenv::dotenv;
#[tokio::main]
async fn main() -> anyhow::Result<()> {
dotenv().ok();
windmill::initialize_tracing().await?;
let db = windmill::connect_db().await?;
let num_workers = std::env::var("NUM_WORKERS")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill::DEFAULT_NUM_WORKERS as i32);
let (server_mode, monitor_mode, migrate_db) = (true, true, true);
if migrate_db {
windmill::migrate_db(&db).await?;
}
let (tx, rx) = tokio::sync::broadcast::channel::<()>(3);
let shutdown_signal = windmill::shutdown_signal(tx.clone());
if server_mode || monitor_mode || num_workers > 0 {
let addr = SocketAddr::from(([0, 0, 0, 0], 8000));
let timeout = std::env::var("TIMEOUT")
.ok()
.and_then(|x| x.parse::<i32>().ok())
.unwrap_or(windmill::DEFAULT_TIMEOUT);
let server_f = async {
if server_mode {
windmill::run_server(
db.clone(),
addr,
&std::env::var("BASE_URL").unwrap_or("http://localhost".to_string()),
windmill::EmailSender {
from: "bot@windmill.dev".to_string(),
server: "smtp.gmail.com".to_string(),
password: std::env::var("SMTP_PASSWORD").unwrap_or("NOPASS".to_string()),
},
rx,
)
.await?;
}
Ok(()) as anyhow::Result<()>
};
let base_url = std::env::var("BASE_INTERNAL_URL")
.unwrap_or_else(|_| "http://missing-base-url".to_string());
let workers_f = async {
if num_workers > 0 {
let sleep_queue = std::env::var("SLEEP_QUEUE")
.ok()
.and_then(|x| x.parse::<u64>().ok())
.unwrap_or(windmill::DEFAULT_SLEEP_QUEUE);
windmill::run_workers(
db.clone(),
addr,
timeout,
num_workers,
sleep_queue,
base_url,
tx.clone(),
)
.await?;
}
Ok(()) as anyhow::Result<()>
};
let monitor_f = async {
if monitor_mode {
windmill::monitor_db(&db, timeout, tx.clone());
}
Ok(()) as anyhow::Result<()>
};
futures::try_join!(shutdown_signal, server_f, workers_f, monitor_f)?;
}
Ok(())
}

712
backend/src/oauth2.rs Normal file
View File

@@ -0,0 +1,712 @@
use std::fmt::Debug;
use std::sync::Arc;
use std::time::Duration;
use axum::body::Bytes;
use axum::extract::{Extension, FromRequest, Path, Query, RequestParts};
use axum::response::Redirect;
use axum::routing::{get, post};
use axum::{async_trait, Router};
use futures::TryFutureExt;
use hyper::StatusCode;
use oauth2::basic::{
BasicClient, BasicErrorResponse, BasicRevocationErrorResponse, BasicTokenIntrospectionResponse,
BasicTokenType,
};
use oauth2::reqwest::async_http_client;
use oauth2::{helpers, TokenType};
use oauth2::{AccessToken, Client as OClient, RefreshToken, StandardRevocableToken};
// Alternatively, this can be `oauth2::curl::http_client` or a custom client.
use oauth2::{
AuthUrl, AuthorizationCode, ClientId, ClientSecret, CsrfToken, RedirectUrl, Scope,
TokenResponse, TokenUrl,
};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use slack_http_verifier::SlackVerifier;
use tower_cookies::{Cookie, Cookies};
use crate::audit::{audit_log, ActionKind};
use crate::db::{UserDB, DB};
use crate::error::{self, to_anyhow, Error, Result};
use crate::jobs::{get_latest_hash_for_path, JobPayload};
use crate::users::{Authed, LoginType};
use crate::variables::build_crypt;
use crate::workspaces::WorkspaceSettings;
use crate::{jobs, BasicClientsMap};
use crate::{variables, BaseUrl};
pub fn global_service() -> Router {
Router::new()
.route("/login/:client", get(login))
.route("/login_callback/:client", get(login_callback))
.route(
"/slack_command",
post(slack_command).route_layer(axum::extract::extractor_middleware::<SlackSig>()),
)
}
pub fn workspaced_service() -> Router {
Router::new()
.route("/connect/:client", get(connect))
.route("/disconnect/:client", post(disconnect))
.route("/connect_callback/:client", get(connect_callback))
}
pub fn build_gh_client(client_id: &str, client_secret: &str, base_uri: &str) -> BasicClient {
let auth_url = AuthUrl::new("https://github.com/login/oauth/authorize".to_string())
.expect("Invalid authorization endpoint URL");
let token_url = TokenUrl::new("https://github.com/login/oauth/access_token".to_string())
.expect("Invalid token endpoint URL");
// Set up the config for the Github OAuth2 process.
BasicClient::new(
ClientId::new(client_id.to_string()),
Some(ClientSecret::new(client_secret.to_string())),
auth_url,
Some(token_url),
)
.set_redirect_uri(
RedirectUrl::new(format!("{base_uri}/api/oauth/login_callback/github")).unwrap(),
)
}
pub fn build_connect_client(w_id: &str, client_name: &str, base_uri: &str) -> Result<BasicClient> {
let (auth_str, token_str) = match client_name {
"gmail" => ("", ""),
"slack" => (
"https://slack.com/oauth/authorize",
"https://slack.com/api/oauth.access",
),
_ => Err(Error::BadRequest(format!("unrecognized client!")))?,
};
let auth_url = AuthUrl::new(auth_str.to_string()).expect("Invalid authorization endpoint URL");
let token_url = TokenUrl::new(token_str.to_string()).expect("Invalid token endpoint URL");
// Set up the config for the Github OAuth2 process.
Ok(BasicClient::new(
ClientId::new(
std::env::var(&format!("{}_OAUTH_CLIENT_ID", client_name.to_uppercase()))
.ok()
.ok_or(Error::BadRequest(format!(
"client id for {} not in env",
client_name
)))?,
),
Some(ClientSecret::new(
std::env::var(&format!(
"{}_OAUTH_CLIENT_SECRET",
client_name.to_uppercase()
))
.ok()
.ok_or(Error::BadRequest(format!(
"client secret for {} not in env",
client_name
)))?,
)),
auth_url,
Some(token_url),
)
.set_redirect_uri(
RedirectUrl::new(format!(
"{base_uri}/api/w/{w_id}/oauth/connect_callback/{client_name}"
))
.unwrap(),
))
}
type SlackClient = OClient<
BasicErrorResponse,
SlackTokenResponse,
BasicTokenType,
BasicTokenIntrospectionResponse,
StandardRevocableToken,
BasicRevocationErrorResponse,
>;
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct SlackTokenResponse {
access_token: AccessToken,
team_id: String,
team_name: String,
#[serde(rename = "scope")]
#[serde(deserialize_with = "helpers::deserialize_space_delimited_vec")]
#[serde(serialize_with = "helpers::serialize_space_delimited_vec")]
#[serde(skip_serializing_if = "Option::is_none")]
#[serde(default)]
scopes: Option<Vec<Scope>>,
bot: SlackBotToken,
}
#[derive(Clone, Debug, Deserialize, Serialize)]
pub struct SlackBotToken {
bot_access_token: String,
}
impl TokenResponse<BasicTokenType> for SlackTokenResponse
where
BasicTokenType: TokenType,
{
///
/// REQUIRED. The access token issued by the authorization server.
///
fn access_token(&self) -> &AccessToken {
&self.access_token
}
///
/// REQUIRED. The type of the token issued as described in
/// [Section 7.1](https://tools.ietf.org/html/rfc6749#section-7.1).
/// Value is case insensitive and deserialized to the generic `TokenType` parameter.
/// But in this particular case as the service is non compliant, it has a default value
///
fn token_type(&self) -> &BasicTokenType {
&BasicTokenType::Bearer
}
///
/// RECOMMENDED. The lifetime in seconds of the access token. For example, the value 3600
/// denotes that the access token will expire in one hour from the time the response was
/// generated. If omitted, the authorization server SHOULD provide the expiration time via
/// other means or document the default value.
///
fn expires_in(&self) -> Option<Duration> {
None
}
///
/// OPTIONAL. The refresh token, which can be used to obtain new access tokens using the same
/// authorization grant as described in
/// [Section 6](https://tools.ietf.org/html/rfc6749#section-6).
///
fn refresh_token(&self) -> Option<&RefreshToken> {
None
}
///
/// OPTIONAL, if identical to the scope requested by the client; otherwise, REQUIRED. The
/// scipe of the access token as described by
/// [Section 3.3](https://tools.ietf.org/html/rfc6749#section-3.3). If included in the response,
/// this space-delimited field is parsed into a `Vec` of individual scopes. If omitted from
/// the response, this field is `None`.
///
fn scopes(&self) -> Option<&Vec<Scope>> {
self.scopes.as_ref()
}
}
pub fn build_slack_client(w_id: &str, client_name: &str, base_uri: &str) -> Result<SlackClient> {
let (auth_str, token_str) = (
"https://slack.com/oauth/authorize",
"https://slack.com/api/oauth.access",
);
let auth_url = AuthUrl::new(auth_str.to_string()).expect("Invalid authorization endpoint URL");
let token_url = TokenUrl::new(token_str.to_string()).expect("Invalid token endpoint URL");
// Set up the config for the Github OAuth2 process.
Ok(SlackClient::new(
ClientId::new(
std::env::var(&format!("{}_OAUTH_CLIENT_ID", client_name.to_uppercase()))
.ok()
.ok_or(Error::BadRequest(format!(
"client id for {} not in env",
client_name
)))?,
),
Some(ClientSecret::new(
std::env::var(&format!(
"{}_OAUTH_CLIENT_SECRET",
client_name.to_uppercase()
))
.ok()
.ok_or(Error::BadRequest(format!(
"client secret for {} not in env",
client_name
)))?,
)),
auth_url,
Some(token_url),
)
.set_redirect_uri(
RedirectUrl::new(format!(
"{base_uri}/api/w/{w_id}/oauth/connect_callback/{client_name}"
))
.unwrap(),
))
}
async fn connect(
Path((w_id, client_name)): Path<(String, String)>,
Extension(base_url): Extension<BaseUrl>,
cookies: Cookies,
) -> error::Result<Redirect> {
let client = build_connect_client(&w_id, &client_name, &base_url.0)?;
let (authorize_url, csrf_state) = client
.authorize_url(CsrfToken::new_random)
.add_scope(Scope::new("bot".to_string()))
.add_scope(Scope::new("commands".to_string()))
.url();
let csrf = csrf_state.secret().to_string();
let mut cookie = Cookie::new("csrf", csrf);
cookie.set_path("/");
cookies.add(cookie);
Ok(Redirect::to(authorize_url.as_str()))
}
async fn disconnect(
authed: Authed,
Path((w_id, client_name)): Path<(String, String)>,
Extension(user_db): Extension<UserDB>,
) -> error::Result<String> {
let mut tx = user_db.begin(&authed).await?;
match client_name.as_str() {
"slack" => {
sqlx::query!(
"UPDATE workspace_settings
SET slack_team_id = null, slack_name = null WHERE workspace_id = $1",
&w_id
)
.execute(&mut tx)
.await?;
}
_ => Err(error::Error::BadRequest(format!(
"Not recognized client name {client_name}"
)))?,
}
tx.commit().await?;
Ok(format!("{client_name} disconnected"))
}
async fn login(
Extension(clients): Extension<Arc<BasicClientsMap>>,
Path(client_name): Path<String>,
cookies: Cookies,
) -> error::Result<Redirect> {
let client = clients
.get(&client_name)
.ok_or(Error::BadRequest(format!("client {} invalid", client_name)))?;
let (authorize_url, csrf_state) = client
.authorize_url(CsrfToken::new_random)
.add_scope(Scope::new("user:email".to_string()))
// .add_scope(Scope::new("read:user".to_string()))
.url();
let csrf = csrf_state.secret().to_string();
let mut cookie = Cookie::new("csrf", csrf);
cookie.set_path("/");
cookies.add(cookie);
Ok(Redirect::to(authorize_url.as_str()))
}
#[derive(Deserialize)]
pub struct CallbackQuery {
code: Option<String>,
state: Option<String>,
error: Option<String>,
}
async fn connect_callback(
authed: Authed,
Path((w_id, client_name)): Path<(String, String)>,
Query(query): Query<CallbackQuery>,
cookies: Cookies,
Extension(user_db): Extension<UserDB>,
Extension(base_url): Extension<BaseUrl>,
) -> error::Result<Redirect> {
if let Some(error) = query.error {
return Ok(Redirect::to(&format!(
"/connection_added?error={}",
urlencoding::encode(&error).into_owned()
)));
}
let code = AuthorizationCode::new(query.code.unwrap());
let state = CsrfToken::new(query.state.unwrap());
let csrf_state = cookies
.get("csrf")
.map(|x| x.value().to_string())
.unwrap_or("".to_string());
if state.secret().to_string() != csrf_state {
return Err(error::Error::BadRequest("csrf did not match".to_string()));
}
let mut tx = user_db.begin(&authed).await?;
let mc = build_crypt(&mut tx, &w_id).await?;
let token_res = match client_name.as_str() {
"slack" => {
let t = build_slack_client(&w_id, &client_name, &base_url.0)?
.exchange_code(code)
.request_async(async_http_client)
.await;
if let Ok(token) = t {
sqlx::query!(
"INSERT INTO workspace_settings
(workspace_id, slack_team_id, slack_name)
VALUES ($1, $2, $3) ON CONFLICT (workspace_id) DO UPDATE SET slack_team_id = $2, slack_name = $3",
&w_id,
token.team_id,
token.team_name
)
.execute(&mut tx)
.await?;
sqlx::query!(
"INSERT INTO group_
(workspace_id, name, summary)
VALUES ($1, $2, $3) ON CONFLICT DO NOTHING",
&w_id,
"slack",
"The group that runs the script triggered by the slack /windmill command.
Share scripts to this group to make them executable from slack and add
members to this group to let them manage the slack related owner space."
)
.execute(&mut tx)
.await?;
Ok(token.bot.bot_access_token.to_owned())
} else {
Err(t.unwrap_err())
}
}
_ => {
build_connect_client(&w_id, &client_name, &base_url.0)?
.exchange_code(code)
.request_async(async_http_client)
.map_ok(|t| t.access_token().secret().to_owned())
.await
}
};
if let Ok(token) = token_res {
tracing::info!("{token}");
let variable_path = &format!("g/all/{}_token", &client_name);
sqlx::query!(
"INSERT INTO variable
(workspace_id, path, value, is_secret, description)
VALUES ($1, $2, $3, true, $4) ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
&w_id,
variable_path,
variables::encrypt(&mc, token.to_string()),
format!("OAuth2 token for {client_name}"),
)
.execute(&mut tx)
.await?;
sqlx::query!(
"INSERT INTO resource
(workspace_id, path, value, description, resource_type)
VALUES ($1, $2, $3, $4, $5) ON CONFLICT (workspace_id, path) DO UPDATE SET value = $3",
&w_id,
variable_path,
serde_json::json!({ "token": format!("$var:{variable_path}") }),
format!("OAuth2 token for {client_name}"),
&client_name
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"oauth2.connect",
ActionKind::Create,
&w_id,
Some(&client_name),
None,
)
.await?;
tx.commit().await?;
Ok(Redirect::to(
format!("/connection_added?client_name={}", &client_name).as_str(),
))
} else {
let error = token_res.unwrap_err().to_string();
Ok(Redirect::to(&format!(
"/connection_added?error={}",
urlencoding::encode(&format!("error fetching token: {error}")).into_owned()
)))
}
}
#[derive(Deserialize, Debug)]
pub struct SlackCommand {
team_id: String,
user_name: String,
text: String,
response_url: String,
}
#[derive(Clone, Debug)]
pub struct SlackSig {
sig: String,
ts: String,
}
#[async_trait]
impl<B> FromRequest<B> for SlackSig
where
B: Send,
{
type Rejection = (StatusCode, String);
async fn from_request(req: &mut RequestParts<B>) -> std::result::Result<Self, Self::Rejection> {
let hm = req.headers();
Ok(Self {
sig: hm
.get("X-Slack-Signature")
.map(|x| x.to_str().unwrap_or(""))
.unwrap_or("")
.to_string(),
ts: hm
.get("X-Slack-Request-Timestamp")
.map(|x| x.to_str().unwrap_or(""))
.unwrap_or("")
.to_string(),
})
}
}
async fn slack_command(
SlackSig { sig, ts }: SlackSig,
Extension(slack_verifier): Extension<Arc<Option<SlackVerifier>>>,
Extension(db): Extension<DB>,
Extension(base_url): Extension<BaseUrl>,
body: Bytes,
) -> error::Result<String> {
let form: SlackCommand = serde_urlencoded::from_bytes(&body)
.map_err(|_| error::Error::BadRequest("invalid payload".to_string()))?;
let body = String::from_utf8_lossy(&body);
if slack_verifier
.as_ref()
.as_ref()
.map(|sv| sv.verify(&ts, &body, &sig).ok())
.flatten()
.is_none()
{
return Err(error::Error::BadRequest("verification failed".to_owned()));
}
let mut tx = db.begin().await?;
let settings = sqlx::query_as!(
WorkspaceSettings,
"SELECT * FROM workspace_settings WHERE slack_team_id = $1",
form.team_id,
)
.fetch_optional(&mut tx)
.await?;
if let Some(settings) = settings {
if let Some(script) = &settings.slack_command_script {
let script_hash =
get_latest_hash_for_path(&mut tx, &settings.workspace_id, script).await?;
let mut map = serde_json::Map::new();
map.insert("text".to_string(), serde_json::Value::String(form.text));
map.insert(
"response_url".to_string(),
serde_json::Value::String(form.response_url),
);
let (uuid, tx) = jobs::push(
tx,
&settings.workspace_id,
JobPayload::ScriptHash {
hash: script_hash,
path: script.to_owned(),
},
Some(map),
&form.user_name,
"g/slack".to_string(),
None,
None,
None,
false,
)
.await?;
tx.commit().await?;
let url = base_url.0;
return Ok(format!("Job launched. See details at {url}/run/{uuid}"));
}
}
return Ok(format!(
"workspace not properly configured (did you set the script to trigger in the settings?)"
));
}
#[derive(Deserialize)]
pub struct UserInfo {
name: Option<String>,
company: Option<String>,
}
async fn login_callback(
Path(client_name): Path<String>,
Query(query): Query<CallbackQuery>,
cookies: Cookies,
Extension(clients): Extension<Arc<BasicClientsMap>>,
Extension(db): Extension<DB>,
) -> error::Result<Redirect> {
if let Some(error) = query.error {
return Ok(Redirect::to(&format!(
"/user/login?error={}",
urlencoding::encode(&error).into_owned()
)));
}
let code = AuthorizationCode::new(query.code.unwrap());
let state = CsrfToken::new(query.state.unwrap());
let csrf_state = cookies
.get("csrf")
.map(|x| x.value().to_string())
.unwrap_or("".to_string());
if state.secret().to_string() != csrf_state {
return Err(error::Error::BadRequest("csrf did not match".to_string()));
}
let client = clients.get(&client_name).unwrap();
// Exchange the code with a token.
let token_res = client
.exchange_code(code)
.request_async(async_http_client)
.await;
if let Ok(token) = token_res {
let token = token.access_token().secret();
let http_client = reqwest::ClientBuilder::new()
.user_agent("windmill/beta")
.build()
.map_err(to_anyhow)?;
let email = get_email(&http_client, &client_name, token).await?;
let mut tx = db.begin().await?;
let login: Option<(String, LoginType, bool)> =
sqlx::query_as("SELECT email, login_type, super_admin FROM password WHERE email = $1")
.bind(&email)
.fetch_optional(&mut tx)
.await?;
if let Some((email, login_type, super_admin)) = login {
let login_type = serde_json::json!(login_type);
if login_type == client_name {
crate::users::create_session_token(&email, super_admin, &mut tx, cookies).await?;
} else {
return Err(error::Error::BadRequest(format!(
"an user with the email associated to this login exists but with a different login type {login_type}")
));
}
} else {
let user = get_user_info(&http_client, &client_name, &token).await?;
sqlx::query(
&format!("INSERT INTO password (email, name, company, login_type, verified) VALUES ($1, $2, $3, '{}', true)", &client_name)
)
.bind(&email)
.bind(&user.name)
.bind(user.company)
.execute(&mut tx)
.await?;
crate::users::create_session_token(&email, false, &mut tx, cookies).await?;
audit_log(
&mut tx,
&email,
"oauth.signup",
ActionKind::Create,
"global",
Some("github"),
None,
)
.await?;
let demo_exists =
sqlx::query_scalar!("SELECT EXISTS(SELECT 1 FROM workspace WHERE id = 'demo')")
.fetch_one(&mut tx)
.await?
.unwrap_or(false);
if demo_exists {
sqlx::query!(
"INSERT INTO workspace_invite
(workspace_id, email, is_admin)
VALUES ('demo', $1, false)",
&email
)
.execute(&mut tx)
.await?;
}
}
tx.commit().await?;
Ok(Redirect::to("/user/workspaces"))
} else {
Ok(Redirect::to(&format!(
"/user/login?error={}",
urlencoding::encode("invalid token").into_owned()
)))
}
}
#[derive(Deserialize)]
pub struct GHEmailInfo {
email: String,
verified: bool,
primary: bool,
}
async fn get_email(http_client: &Client, client_name: &str, token: &str) -> error::Result<String> {
let email = match client_name {
"github" => http_client
.get("https://api.github.com/user/emails")
.bearer_auth(token)
.send()
.await
.map_err(to_anyhow)?
.json::<Vec<GHEmailInfo>>()
.await
.map_err(to_anyhow)?
.iter()
.find(|x| x.primary && x.verified)
.ok_or(error::Error::BadRequest(format!(
"user does not have any primary and verified address"
)))?
.email
.to_string(),
_ => {
return Err(error::Error::BadRequest(
"client name not recognized".to_string(),
))
}
};
Ok(email)
}
async fn get_user_info(
http_client: &Client,
client_name: &str,
token: &str,
) -> error::Result<UserInfo> {
let email = match client_name {
"github" => http_client
.get("https://api.github.com/user")
.bearer_auth(token)
.send()
.await
.map_err(to_anyhow)?
.json::<UserInfo>()
.await
.map_err(to_anyhow)?,
_ => {
return Err(error::Error::BadRequest(
"client name not recognized".to_string(),
))
}
};
Ok(email)
}

592
backend/src/parser.rs Normal file
View File

@@ -0,0 +1,592 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::collections::HashMap;
use itertools::Itertools;
use regex::Regex;
use serde::Serialize;
use serde_json::json;
use crate::error;
use rustpython_parser::{
ast::{ExpressionType, Located, Number, StatementType, StringGroup, Varargs},
parser,
};
#[derive(Serialize)]
pub struct MainArgSignature {
pub star_args: bool,
pub star_kwargs: bool,
pub args: Vec<Arg>,
}
#[derive(Serialize)]
#[serde(rename_all(serialize = "lowercase"))]
pub enum Typ {
Str,
Int,
Float,
Bool,
Dict,
List,
Bytes,
Datetime,
Unknown,
}
#[derive(Serialize)]
pub struct Arg {
pub name: String,
pub typ: Typ,
pub default: Option<serde_json::Value>,
pub has_default: bool,
}
pub fn parse_signature(code: &str) -> error::Result<MainArgSignature> {
let ast = parser::parse_program(code)
.map_err(|e| error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string())))?
.statements;
let param = ast.into_iter().find_map(|x| match x {
Located {
location: _,
node:
StatementType::FunctionDef {
is_async: _,
name,
args,
body: _,
decorator_list: _,
returns: _,
},
} if &name == "main" => Some(*args),
_ => None,
});
if let Some(params) = param {
//println!("{:?}", params);
let def_arg_start = params.args.len() - params.defaults.len();
Ok(MainArgSignature {
star_args: params.vararg != Varargs::None,
star_kwargs: params.vararg != Varargs::None,
args: params
.args
.into_iter()
.enumerate()
.map(|(i, x)| {
let default = if i >= def_arg_start {
to_value(&params.defaults[i - def_arg_start].node)
} else {
None
};
Arg {
name: x.arg,
typ: x.annotation.map_or(Typ::Unknown, |e| match *e {
Located {
location: _,
node: ExpressionType::Identifier { name },
} => match name.as_ref() {
"str" => Typ::Str,
"float" => Typ::Float,
"int" => Typ::Int,
"bool" => Typ::Bool,
"dict" => Typ::Dict,
"list" => Typ::List,
"bytes" => Typ::Bytes,
"datetime" => Typ::Datetime,
"datetime.datetime" => Typ::Datetime,
_ => Typ::Unknown,
},
_ => Typ::Unknown,
}),
has_default: default.is_some(),
default,
}
})
.collect(),
})
} else {
Err(error::Error::ExecutionErr(
"main function was not findable".to_string(),
))
}
}
const STDIMPORTS: [&str; 301] = [
"__future__",
"_abc",
"_aix_support",
"_ast",
"_asyncio",
"_bisect",
"_blake2",
"_bootsubprocess",
"_bz2",
"_codecs",
"_codecs_cn",
"_codecs_hk",
"_codecs_iso2022",
"_codecs_jp",
"_codecs_kr",
"_codecs_tw",
"_collections",
"_collections_abc",
"_compat_pickle",
"_compression",
"_contextvars",
"_crypt",
"_csv",
"_ctypes",
"_curses",
"_curses_panel",
"_datetime",
"_dbm",
"_decimal",
"_elementtree",
"_frozen_importlib",
"_frozen_importlib_external",
"_functools",
"_gdbm",
"_hashlib",
"_heapq",
"_imp",
"_io",
"_json",
"_locale",
"_lsprof",
"_lzma",
"_markupbase",
"_md5",
"_msi",
"_multibytecodec",
"_multiprocessing",
"_opcode",
"_operator",
"_osx_support",
"_overlapped",
"_pickle",
"_posixshmem",
"_posixsubprocess",
"_py_abc",
"_pydecimal",
"_pyio",
"_queue",
"_random",
"_sha1",
"_sha256",
"_sha3",
"_sha512",
"_signal",
"_sitebuiltins",
"_socket",
"_sqlite3",
"_sre",
"_ssl",
"_stat",
"_statistics",
"_string",
"_strptime",
"_struct",
"_symtable",
"_thread",
"_threading_local",
"_tkinter",
"_tracemalloc",
"_uuid",
"_warnings",
"_weakref",
"_weakrefset",
"_winapi",
"_zoneinfo",
"abc",
"aifc",
"antigravity",
"argparse",
"array",
"ast",
"asynchat",
"asyncio",
"asyncore",
"atexit",
"audioop",
"base64",
"bdb",
"binascii",
"binhex",
"bisect",
"builtins",
"bz2",
"cProfile",
"calendar",
"cgi",
"cgitb",
"chunk",
"cmath",
"cmd",
"code",
"codecs",
"codeop",
"collections",
"colorsys",
"compileall",
"concurrent",
"configparser",
"contextlib",
"contextvars",
"copy",
"copyreg",
"crypt",
"csv",
"ctypes",
"curses",
"dataclasses",
"datetime",
"dbm",
"decimal",
"difflib",
"dis",
"distutils",
"doctest",
"email",
"encodings",
"ensurepip",
"enum",
"errno",
"faulthandler",
"fcntl",
"filecmp",
"fileinput",
"fnmatch",
"fractions",
"ftplib",
"functools",
"gc",
"genericpath",
"getopt",
"getpass",
"gettext",
"glob",
"graphlib",
"grp",
"gzip",
"hashlib",
"heapq",
"hmac",
"html",
"http",
"idlelib",
"imaplib",
"imghdr",
"imp",
"importlib",
"inspect",
"io",
"ipaddress",
"itertools",
"json",
"keyword",
"lib2to3",
"linecache",
"locale",
"logging",
"lzma",
"mailbox",
"mailcap",
"marshal",
"math",
"mimetypes",
"mmap",
"modulefinder",
"msilib",
"msvcrt",
"multiprocessing",
"netrc",
"nis",
"nntplib",
"nt",
"ntpath",
"nturl2path",
"numbers",
"opcode",
"operator",
"optparse",
"os",
"ossaudiodev",
"pathlib",
"pdb",
"pickle",
"pickletools",
"pipes",
"pkgutil",
"platform",
"plistlib",
"poplib",
"posix",
"posixpath",
"pprint",
"profile",
"pstats",
"pty",
"pwd",
"py_compile",
"pyclbr",
"pydoc",
"pydoc_data",
"pyexpat",
"queue",
"quopri",
"random",
"re",
"readline",
"reprlib",
"resource",
"rlcompleter",
"runpy",
"sched",
"secrets",
"select",
"selectors",
"shelve",
"shlex",
"shutil",
"signal",
"site",
"smtpd",
"smtplib",
"sndhdr",
"socket",
"socketserver",
"spwd",
"sqlite3",
"sre_compile",
"sre_constants",
"sre_parse",
"ssl",
"stat",
"statistics",
"string",
"stringprep",
"struct",
"subprocess",
"sunau",
"symtable",
"sys",
"sysconfig",
"syslog",
"tabnanny",
"tarfile",
"telnetlib",
"tempfile",
"termios",
"textwrap",
"this",
"threading",
"time",
"timeit",
"tkinter",
"token",
"tokenize",
"trace",
"traceback",
"tracemalloc",
"tty",
"turtle",
"turtledemo",
"types",
"typing",
"unicodedata",
"unittest",
"urllib",
"uu",
"uuid",
"venv",
"warnings",
"wave",
"weakref",
"webbrowser",
"winreg",
"winsound",
"wsgiref",
"xdrlib",
"xml",
"xmlrpc",
"zipapp",
"zipfile",
"zipimport",
"",
];
fn to_value(et: &ExpressionType) -> Option<serde_json::Value> {
match et {
ExpressionType::String {
value: StringGroup::Constant { value },
} => Some(json!(value)),
ExpressionType::Number { value } => match value {
Number::Integer { value } => Some(json!(value.to_string().parse::<i64>().unwrap())),
Number::Float { value } => Some(json!(value)),
_ => None,
},
ExpressionType::True => Some(json!(true)),
ExpressionType::False => Some(json!(false)),
ExpressionType::Dict { elements } => {
let v = elements
.into_iter()
.map(|(k, v)| {
let key = k
.as_ref()
.and_then(|x| to_value(&x.node))
.and_then(|x| match x {
serde_json::Value::String(s) => Some(s),
_ => None,
})
.unwrap_or_else(|| "no_key".to_string());
(key, to_value(&v.node))
})
.collect::<HashMap<String, _>>();
Some(json!(v))
}
ExpressionType::List { elements } => {
let v = elements
.into_iter()
.map(|x| to_value(&x.node))
.collect::<Vec<_>>();
Some(json!(v))
}
ExpressionType::None => Some(json!(null)),
ExpressionType::Call {
function: _,
args: _,
keywords: _,
} => Some(json!("<function call>")),
_ => None,
}
}
pub fn parse_imports(code: &str) -> error::Result<Vec<String>> {
let find_requirements = code
.lines()
.find_position(|x| x.starts_with("#requirements:"));
let re = Regex::new(r"^\#(\S+)$").unwrap();
if let Some((pos, _)) = find_requirements {
let lines = code
.lines()
.skip(pos + 1)
.map_while(|x| {
re.captures(x)
.map(|x| x.get(1).unwrap().as_str().to_string())
})
.collect();
Ok(lines)
} else {
let ast = parser::parse_program(code)
.map_err(|e| {
error::Error::ExecutionErr(format!("Error parsing code: {}", e.to_string()))
})?
.statements;
let imports = ast
.into_iter()
.filter_map(|x| match x {
Located { location: _, node } => match node {
StatementType::Import { names } => Some(
names
.into_iter()
.map(|x| x.symbol.split('.').next().unwrap_or("").to_string())
.collect::<Vec<String>>(),
),
StatementType::ImportFrom {
level: _,
module: Some(mod_),
names: _,
} => Some(vec![mod_
.split('.')
.next()
.unwrap_or("")
.to_string()
.replace("_", "-")]),
_ => None,
},
})
.flatten()
.filter(|x| !STDIMPORTS.contains(&x.as_str()))
.unique()
.collect();
Ok(imports)
}
}
#[cfg(test)]
mod tests {
// Note this useful idiom: importing names from outer (for mod tests) scope.
use super::*;
#[test]
fn test_parse_sig() -> anyhow::Result<()> {
//let code = "print(2 + 3, fd=sys.stderr)";
let code = "
import os
def main(test1: str, name: datetime.datetime = datetime.now(), byte: bytes = bytes(1)):
print(f\"Hello World and a warm welcome especially to {name}\")
print(\"The env variable at `all/pretty_secret`: \", os.environ.get(\"ALL_PRETTY_SECRET\"))
return {\"len\": len(name), \"splitted\": name.split() }
";
println!("{}", serde_json::to_string(&parse_signature(code)?)?);
Ok(())
}
#[test]
fn test_parse_imports() -> anyhow::Result<()> {
//let code = "print(2 + 3, fd=sys.stderr)";
let code = "
import os
import wmill
from zanzibar.estonie import talin
import matplotlib.pyplot as plt
def main():
pass
";
let r = parse_imports(code)?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["wmill", "zanzibar", "matplotlib"]);
Ok(())
}
#[test]
fn test_parse_imports2() -> anyhow::Result<()> {
//let code = "print(2 + 3, fd=sys.stderr)";
let code = "
#requirements:
#burkina=0.4
#nigeria
#
#congo
import os
import wmill
from zanzibar.estonie import talin
def main():
pass
";
let r = parse_imports(code)?;
println!("{}", serde_json::to_string(&r)?);
assert_eq!(r, vec!["burkina=0.4", "nigeria"]);
Ok(())
}
}

426
backend/src/resources.rs Normal file
View File

@@ -0,0 +1,426 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
users::Authed,
utils::{require_admin, Pagination, StripPath},
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use hyper::StatusCode;
use serde::{Deserialize, Serialize};
use sql_builder::{bind::Bind, SqlBuilder};
use sqlx::FromRow;
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_resources))
.route("/get/*path", get(get_resource))
.route("/get_value/*path", get(get_resource_value))
.route("/update/*path", post(update_resource))
.route("/delete/*path", delete(delete_resource))
.route("/create", post(create_resource))
.route("/type/list", get(list_resource_types))
.route("/type/listnames", get(list_resource_types_names))
.route("/type/get/:name", get(get_resource_type))
.route("/type/update/:name", post(update_resource_type))
.route("/type/delete/:name", delete(delete_resource_type))
.route("/type/create", post(create_resource_type))
}
#[derive(FromRow, Serialize, Deserialize)]
pub struct ResourceType {
pub workspace_id: String,
pub name: String,
pub schema: Option<serde_json::Value>,
pub description: Option<String>,
}
#[derive(Deserialize)]
pub struct CreateResourceType {
pub name: String,
pub schema: Option<serde_json::Value>,
pub description: Option<String>,
}
#[derive(Deserialize)]
pub struct EditResourceType {
pub schema: Option<serde_json::Value>,
pub description: Option<String>,
}
#[derive(FromRow, Serialize, Deserialize)]
pub struct Resource {
pub workspace_id: String,
pub path: String,
pub value: Option<serde_json::Value>,
pub description: Option<String>,
pub resource_type: String,
pub extra_perms: serde_json::Value,
}
#[derive(Deserialize)]
pub struct CreateResource {
pub path: String,
pub value: Option<serde_json::Value>,
pub description: Option<String>,
pub resource_type: String,
}
#[derive(Deserialize)]
struct EditResource {
path: Option<String>,
description: Option<String>,
value: Option<serde_json::Value>,
}
#[derive(Deserialize)]
pub struct ListResourceQuery {
resource_type: Option<String>,
}
async fn list_resources(
authed: Authed,
Query(lq): Query<ListResourceQuery>,
Query(pagination): Query<Pagination>,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<Resource>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let mut sqlb = SqlBuilder::select_from("resource")
.fields(&[
"workspace_id",
"path",
"null::JSONB as value",
"description",
"resource_type",
"extra_perms",
])
.order_by("path", true)
.and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
.offset(offset)
.limit(per_page)
.clone();
if let Some(rt) = &lq.resource_type {
sqlb.and_where_eq("resource_type", "?".bind(rt));
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as::<_, Resource>(&sql)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
}
async fn get_resource(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Resource> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let resource_o = sqlx::query_as!(
Resource,
"SELECT * from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
path.to_owned(),
&w_id
)
.fetch_optional(&mut tx)
.await?;
tx.commit().await?;
let resource = crate::utils::not_found_if_none(resource_o, "Resource", path)?;
Ok(Json(resource))
}
async fn get_resource_value(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Option<serde_json::Value>> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let value_o = sqlx::query_scalar!(
"SELECT value from resource WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
path.to_owned(),
&w_id
)
.fetch_optional(&mut tx)
.await?;
tx.commit().await?;
let value = crate::utils::not_found_if_none(value_o, "Resource", path)?;
Ok(Json(value))
}
async fn create_resource(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(resource): Json<CreateResource>,
) -> Result<(StatusCode, String)> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"INSERT INTO resource
(workspace_id, path, value, description, resource_type)
VALUES ($1, $2, $3, $4, $5)",
w_id,
resource.path,
resource.value,
resource.description,
resource.resource_type,
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"resources.create",
ActionKind::Create,
&w_id,
Some(&resource.path),
None,
)
.await?;
tx.commit().await?;
Ok((
StatusCode::CREATED,
format!("resource {} created", resource.path),
))
}
async fn delete_resource(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"DELETE FROM resource WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"resources.delete",
ActionKind::Delete,
&w_id,
Some(path),
None,
)
.await?;
tx.commit().await?;
Ok(format!("resource {} deleted", path))
}
async fn update_resource(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(ns): Json<EditResource>,
) -> Result<String> {
use sql_builder::prelude::*;
let path = path.to_path();
let mut sqlb = SqlBuilder::update_table("resource");
sqlb.and_where_eq("path", "?".bind(&path));
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
if let Some(npath) = &ns.path {
sqlb.set_str("path", npath);
}
if let Some(nvalue) = ns.value {
sqlb.set_str("value", nvalue.to_string());
}
if let Some(ndesc) = ns.description {
sqlb.set_str("description", ndesc);
}
let mut tx = user_db.begin(&authed).await?;
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
sqlx::query(&sql).execute(&mut tx).await?;
audit_log(
&mut tx,
&authed.username,
"resources.update",
ActionKind::Update,
&w_id,
Some(path),
None,
)
.await?;
tx.commit().await?;
Ok(format!("resource {} updated (npath: {:?})", path, ns.path))
}
async fn list_resource_types(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<ResourceType>> {
let rows = sqlx::query_as!(ResourceType, "SELECT * from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
.fetch_all(&db)
.await?;
Ok(Json(rows))
}
async fn list_resource_types_names(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<String>> {
let rows = sqlx::query_scalar!("SELECT name from resource_type WHERE (workspace_id = $1 OR workspace_id = 'starter') ORDER BY name", &w_id)
.fetch_all(&db)
.await?;
Ok(Json(rows))
}
async fn get_resource_type(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
) -> JsonResult<ResourceType> {
let mut tx = user_db.begin(&authed).await?;
let resource_type_o = sqlx::query_as!(
ResourceType,
"SELECT * from resource_type WHERE name = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
&name,
&w_id
)
.fetch_optional(&mut tx)
.await?;
tx.commit().await?;
let resource_type = crate::utils::not_found_if_none(resource_type_o, "ResourceType", name)?;
Ok(Json(resource_type))
}
async fn create_resource_type(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(resource_type): Json<CreateResourceType>,
) -> Result<(StatusCode, String)> {
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"INSERT INTO resource_type
(workspace_id, name, schema, description)
VALUES ($1, $2, $3, $4)",
w_id,
resource_type.name,
resource_type.schema,
resource_type.description,
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"resource_types.create",
ActionKind::Create,
&w_id,
Some(&resource_type.name),
None,
)
.await?;
tx.commit().await?;
Ok((
StatusCode::CREATED,
format!("resource_type {} created", resource_type.name),
))
}
async fn delete_resource_type(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
) -> Result<String> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"DELETE FROM resource_type WHERE name = $1 AND workspace_id = $2",
name,
w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"resource_types.delete",
ActionKind::Delete,
&w_id,
Some(&name),
None,
)
.await?;
tx.commit().await?;
Ok(format!("resource_type {} deleted", name))
}
async fn update_resource_type(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, name)): Path<(String, String)>,
Json(ns): Json<EditResourceType>,
) -> Result<String> {
use sql_builder::prelude::*;
let mut sqlb = SqlBuilder::update_table("resource_type");
sqlb.and_where_eq("name", "?".bind(&name));
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
if let Some(nschema) = ns.schema {
sqlb.set_str("schema", nschema);
}
if let Some(ndesc) = ns.description {
sqlb.set_str("description", ndesc);
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
sqlx::query(&sql).execute(&mut tx).await?;
audit_log(
&mut tx,
&authed.username,
"resource_types.update",
ActionKind::Update,
&w_id,
Some(&name),
None,
)
.await?;
tx.commit().await?;
Ok(format!("resource_type {} updated", name))
}

360
backend/src/schedule.rs Normal file
View File

@@ -0,0 +1,360 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use std::str::FromStr;
use crate::{
audit::{audit_log, ActionKind},
db::UserDB,
error::{self, JsonResult, Result},
jobs::{self, push, JobPayload},
users::Authed,
utils::{get_owner_from_path, Pagination, StripPath},
};
use axum::{
extract::{Extension, Path, Query},
routing::{get, post},
Json, Router,
};
use chrono::{DateTime, Duration, FixedOffset};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use sqlx::{FromRow, Postgres, Transaction};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_schedule))
.route("/get/*path", get(get_schedule))
.route("/create", post(create_schedule))
.route("/update/*path", post(edit_schedule))
.route("/setenabled/*path", post(set_enabled))
}
pub fn global_service() -> Router {
Router::new().route("/preview", post(preview_schedule))
}
#[derive(FromRow, Serialize, Deserialize, Debug)]
pub struct Schedule {
pub workspace_id: String,
pub path: String,
pub edited_by: String,
pub edited_at: DateTime<chrono::Utc>,
pub schedule: String,
pub offset_: i32,
pub enabled: bool,
pub script_path: String,
pub is_flow: bool,
pub args: Option<serde_json::Value>,
pub extra_perms: serde_json::Value,
}
#[derive(Deserialize)]
pub struct NewSchedule {
pub path: String,
pub schedule: String,
pub offset: i32,
pub script_path: String,
pub is_flow: bool,
pub args: Option<serde_json::Value>,
}
pub async fn push_scheduled_job<'c>(
mut tx: Transaction<'c, Postgres>,
schedule: Schedule,
) -> Result<Transaction<'c, Postgres>> {
let sched = cron::Schedule::from_str(&schedule.schedule)
.map_err(|e| error::Error::BadRequest(e.to_string()))?;
let offset = Duration::minutes(schedule.offset_.into());
let next = sched
.after(&(chrono::Utc::now() - offset + Duration::seconds(1)))
.next()
.expect("a schedule should have a next event")
+ offset;
let mut args: Option<Map<String, Value>> = None;
if let Some(args_v) = schedule.args {
if let Value::Object(args_m) = args_v {
args = Some(args_m)
} else {
return Err(error::Error::ExecutionErr(
"args of scripts needs to be dict".to_string(),
));
}
}
let payload = if schedule.is_flow {
JobPayload::Flow(schedule.script_path)
} else {
JobPayload::ScriptHash {
hash: jobs::get_latest_hash_for_path(
&mut tx,
&schedule.workspace_id,
&schedule.script_path,
)
.await?,
path: schedule.script_path,
}
};
let (_, tx) = push(
tx,
&schedule.workspace_id,
payload,
args,
&schedule_to_user(&schedule.path),
get_owner_from_path(&schedule.path),
Some(next),
Some(schedule.path),
None,
false,
)
.await?;
Ok(tx)
}
async fn create_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(ns): Json<NewSchedule>,
) -> Result<String> {
cron::Schedule::from_str(&ns.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let schedule = sqlx::query_as!(Schedule,
"INSERT INTO schedule (workspace_id, path, schedule, offset_, edited_by, script_path, is_flow, args) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *",
w_id,
ns.path,
ns.schedule,
ns.offset,
&authed.username,
ns.script_path,
ns.is_flow,
ns.args
)
.fetch_one(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"schedule.create",
ActionKind::Create,
&w_id,
Some(&ns.path.to_string()),
Some(
[
Some(("schedule", ns.schedule.as_str())),
Some(("script_path", ns.script_path.as_str())),
]
.into_iter()
.flatten()
.collect(),
),
)
.await?;
let tx = push_scheduled_job(tx, schedule).await?;
tx.commit().await?;
Ok(ns.path.to_string())
}
#[derive(Deserialize)]
pub struct EditSchedule {
pub schedule: String,
pub script_path: String,
pub is_flow: bool,
pub args: Option<serde_json::Value>,
}
async fn clear_schedule<'c>(db: &mut Transaction<'c, Postgres>, path: &str) -> Result<()> {
sqlx::query!("DELETE FROM queue WHERE schedule_path = $1", path)
.execute(db)
.await?;
Ok(())
}
async fn edit_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(es): Json<EditSchedule>,
) -> Result<String> {
let path = path.to_path();
cron::Schedule::from_str(&es.schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
clear_schedule(&mut tx, path).await?;
let schedule = sqlx::query_as!(Schedule,
"UPDATE schedule SET schedule = $1, script_path = $2, is_flow = $3, args = $4 WHERE path = $5 AND workspace_id = $6 RETURNING *",
es.schedule,
es.script_path,
es.is_flow,
es.args,
path,
w_id,
)
.fetch_one(&mut tx)
.await?;
if schedule.enabled {
tx = push_scheduled_job(tx, schedule).await?;
}
audit_log(
&mut tx,
&authed.username,
"schedule.edit",
ActionKind::Update,
&w_id,
Some(&path.to_string()),
Some(
[
Some(("schedule", es.schedule.as_str())),
Some(("script_path", es.script_path.as_str())),
]
.into_iter()
.flatten()
.collect(),
),
)
.await?;
tx.commit().await?;
Ok(path.to_string())
}
async fn list_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
) -> JsonResult<Vec<Schedule>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as!(
Schedule,
"SELECT * FROM schedule WHERE workspace_id = $1 ORDER BY edited_at desc LIMIT $2 OFFSET $3",
w_id,
per_page as i64,
offset as i64
)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
}
pub async fn get_schedule_opt<'c>(
db: &mut Transaction<'c, Postgres>,
w_id: &str,
path: &str,
) -> Result<Option<Schedule>> {
let schedule_opt = sqlx::query_as!(
Schedule,
"SELECT * FROM schedule WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.fetch_optional(db)
.await?;
Ok(schedule_opt)
}
async fn get_schedule(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Schedule> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let schedule_o = get_schedule_opt(&mut tx, &w_id, path).await?;
let schedule = crate::utils::not_found_if_none(schedule_o, "Schedule", path)?;
tx.commit().await?;
Ok(Json(schedule))
}
#[derive(Deserialize)]
pub struct PreviewPayload {
pub schedule: String,
pub offset: Option<i32>,
}
pub async fn preview_schedule(
Json(PreviewPayload { schedule, offset }): Json<PreviewPayload>,
) -> JsonResult<Vec<DateTime<chrono::Utc>>> {
let schedule =
cron::Schedule::from_str(&schedule).map_err(|e| error::Error::BadRequest(e.to_string()))?;
let upcoming: Vec<DateTime<chrono::Utc>> = schedule
.upcoming(get_offset(offset))
.take(10)
.map(|x| x.into())
.collect();
Ok(Json(upcoming))
}
fn get_offset(offset: Option<i32>) -> FixedOffset {
FixedOffset::west(offset.unwrap_or(0) * 60)
}
#[derive(Deserialize)]
pub struct SetEnabled {
pub enabled: bool,
}
pub async fn set_enabled(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(SetEnabled { enabled }): Json<SetEnabled>,
) -> Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let schedule_o = sqlx::query_as!(
Schedule,
"UPDATE schedule SET enabled = $1 WHERE path = $2 AND workspace_id = $3 RETURNING *",
enabled,
path,
w_id
)
.fetch_optional(&mut tx)
.await?;
let schedule = crate::utils::not_found_if_none(schedule_o, "Schedule", path)?;
clear_schedule(&mut tx, path).await?;
if enabled {
tx = push_scheduled_job(tx, schedule).await?;
}
audit_log(
&mut tx,
&authed.username,
"schedule.setenabled",
ActionKind::Update,
&w_id,
Some(path),
Some([("enabled", enabled.to_string().as_ref())].into()),
)
.await?;
tx.commit().await?;
Ok(format!(
"succesfully updated schedule at path {} to status {}",
path, enabled
))
}
fn schedule_to_user(path: &str) -> String {
format!("schedule-{}", path.replace('/', "-"))
}

614
backend/src/scripts.rs Normal file
View File

@@ -0,0 +1,614 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use serde::Deserializer;
use sql_builder::prelude::*;
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
jobs, parser,
users::{owner_to_token_owner, truncate_token, Authed, Tokened},
utils::{require_admin, Pagination, StripPath},
};
use axum::{
extract::{Extension, Path, Query},
routing::{get, post},
Json, Router,
};
use hyper::StatusCode;
use serde::{de::Error as _, ser::SerializeSeq, Deserialize, Serialize};
use serde_json::{json, to_string_pretty};
use sql_builder::SqlBuilder;
use sqlx::{FromRow, Postgres, Transaction};
use std::{
collections::hash_map::DefaultHasher,
fmt::Display,
hash::{Hash, Hasher},
};
const MAX_HASH_HISTORY_LENGTH_STORED: usize = 20;
pub fn global_service() -> Router {
Router::new().route("/tojsonschema", post(parse_code_to_jsonschema))
}
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_scripts))
.route("/create", post(create_script))
.route("/archive/p/*path", post(archive_script_by_path))
.route("/get/p/*path", get(get_script_by_path))
.route("/archive/h/:hash", post(archive_script_by_hash))
.route("/delete/h/:hash", post(delete_script_by_hash))
.route("/get/h/:hash", get(get_script_by_hash))
.route("/deployment_status/h/:hash", get(get_deployment_status))
}
#[derive(sqlx::Type, PartialEq, Debug, Hash, Clone, Copy)]
#[sqlx(transparent)]
pub struct ScriptHash(pub i64);
#[derive(sqlx::Type, PartialEq)]
#[sqlx(transparent)]
pub struct ScriptHashes(Vec<i64>);
impl Display for ScriptHash {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "{}", to_hex_string(&self.0))
}
}
impl Serialize for ScriptHash {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(to_hex_string(&self.0).as_str())
}
}
impl<'de> Deserialize<'de> for ScriptHash {
fn deserialize<D>(deserializer: D) -> std::result::Result<ScriptHash, D::Error>
where
D: Deserializer<'de>,
{
let s = String::deserialize(deserializer)?;
let i = to_i64(&s).map_err(|e| D::Error::custom(format!("{}", e)))?;
Ok(ScriptHash(i))
}
}
impl Serialize for ScriptHashes {
fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
let mut seq = serializer.serialize_seq(Some(self.0.len()))?;
for element in &self.0 {
seq.serialize_element(&ScriptHash(*element))?;
}
seq.end()
}
}
#[derive(FromRow, Serialize)]
pub struct Script {
pub workspace_id: String,
pub hash: ScriptHash,
pub path: String,
pub parent_hashes: Option<ScriptHashes>,
pub summary: String,
pub description: String,
pub content: String,
pub created_by: String,
pub created_at: chrono::DateTime<chrono::Utc>,
pub archived: bool,
pub schema: Option<Schema>,
pub deleted: bool,
pub is_template: bool,
pub extra_perms: serde_json::Value,
pub lock: Option<String>,
pub lock_error_logs: Option<String>,
}
#[derive(Serialize, Deserialize, sqlx::Type, Debug)]
#[sqlx(transparent)]
#[serde(transparent)]
pub struct Schema(pub serde_json::Value);
impl Hash for Schema {
fn hash<H: Hasher>(&self, state: &mut H) {
if let Ok(s) = to_string_pretty(&self.0) {
s.hash(state);
}
}
}
#[derive(Serialize, Deserialize, Hash)]
pub struct NewScript {
pub path: String,
pub parent_hash: Option<ScriptHash>,
pub summary: String,
pub description: String,
pub content: String,
pub schema: Option<Schema>,
pub is_template: Option<bool>,
pub lock: Option<Vec<String>>,
}
#[derive(Deserialize)]
pub struct ListScriptQuery {
pub path_start: Option<String>,
pub path_exact: Option<String>,
pub created_by: Option<String>,
pub first_parent_hash: Option<ScriptHash>,
pub last_parent_hash: Option<ScriptHash>,
pub parent_hash: Option<ScriptHash>,
pub show_archived: Option<bool>,
pub order_by: Option<String>,
pub order_desc: Option<bool>,
pub is_template: Option<bool>,
}
async fn list_scripts(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Query(pagination): Query<Pagination>,
Query(lq): Query<ListScriptQuery>,
) -> JsonResult<Vec<Script>> {
let (per_page, offset) = crate::utils::paginate(pagination);
let mut sqlb = SqlBuilder::select_from("script as o")
.fields(&[
"workspace_id",
"hash",
"path",
"array_remove(array[parent_hashes[1]], NULL) as parent_hashes",
"summary",
"description",
"'' as content",
"created_by",
"created_at",
"archived",
"schema",
"deleted",
"is_template",
"extra_perms",
"null as lock",
"CASE WHEN lock_error_logs IS NOT NULL THEN 'error' ELSE null END as lock_error_logs",
])
.order_by("created_at", lq.order_desc.unwrap_or(true))
.and_where("workspace_id = ? OR workspace_id = 'starter'".bind(&w_id))
.offset(offset)
.limit(per_page)
.clone();
if lq.show_archived.unwrap_or(false) {
sqlb.and_where_eq(
"created_at",
"(select max(created_at) from script where o.path = path
AND (workspace_id = $1 OR workspace_id = 'starter'))",
);
} else {
sqlb.and_where_eq("archived", false);
}
if let Some(ps) = &lq.path_start {
sqlb.and_where_like_left("path", "?".bind(ps));
}
if let Some(p) = &lq.path_exact {
sqlb.and_where_eq("path", "?".bind(p));
}
if let Some(cb) = &lq.created_by {
sqlb.and_where_eq("created_by", "?".bind(cb));
}
if let Some(ph) = &lq.first_parent_hash {
sqlb.and_where_eq("parent_hashes[1]", &ph.0);
}
if let Some(ph) = &lq.last_parent_hash {
sqlb.and_where_eq("parent_hashes[array_upper(parent_hashes, 1)]", &ph.0);
}
if let Some(ph) = &lq.parent_hash {
sqlb.and_where_eq("any(parent_hashes)", &ph.0);
}
if let Some(it) = &lq.is_template {
sqlb.and_where_eq("is_template", it);
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as::<_, Script>(&sql).fetch_all(&mut tx).await?;
tx.commit().await?;
Ok(Json(rows))
}
fn hash_script(ns: &NewScript) -> i64 {
let mut dh = DefaultHasher::new();
ns.hash(&mut dh);
dh.finish() as i64
}
async fn create_script(
authed: Authed,
Tokened { token }: Tokened,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(ns): Json<NewScript>,
) -> Result<(StatusCode, String)> {
let hash = ScriptHash(hash_script(&ns));
let mut tx = user_db.begin(&authed).await?;
if sqlx::query_scalar!(
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
hash.0,
&w_id
)
.fetch_optional(&mut tx)
.await?
.is_some()
{
return Err(Error::BadRequest(
"A script with same hash (hence same path, description, summary, content) already \
exists!"
.to_owned(),
));
};
let clashing_script = sqlx::query_as::<_, Script>(
"SELECT * FROM script WHERE path = $1 AND archived = false AND workspace_id = $2",
)
.bind(&ns.path)
.bind(&w_id)
.fetch_optional(&mut tx)
.await?;
let parent_hashes_and_perms: Option<(Vec<i64>, serde_json::Value)> =
match (&ns.parent_hash, clashing_script) {
(None, None) => Ok(None),
(None, Some(s)) => Err(Error::BadRequest(format!(
"Path conflict for {} with non-archived hash {}",
&ns.path, &s.hash
))),
(Some(p_hash), o) => {
if sqlx::query_scalar!(
"SELECT 1 FROM script WHERE hash = $1 AND workspace_id = $2",
p_hash.0,
&w_id
)
.fetch_optional(&mut tx)
.await?
.is_none()
{
return Err(Error::BadRequest(
"The parent hash does not seem to exist".to_owned(),
));
};
let clashing_hash_o = sqlx::query_scalar!(
"SELECT hash FROM script WHERE parent_hashes[1] = $1 AND workspace_id = $2",
p_hash.0,
&w_id
)
.fetch_optional(&mut tx)
.await?;
if let Some(clashing_hash) = clashing_hash_o {
return Err(Error::BadRequest(format!(
"A script with hash {} with same parent_hash has been found. However, the \
lineage must be linear: no 2 scripts can have the same parent",
ScriptHash(clashing_hash)
)));
};
let ps = get_script_by_hash_internal(&mut tx, &w_id, p_hash).await?;
let ph = {
let v = ps.parent_hashes.map(|x| x.0).unwrap_or_default();
let mut v: Vec<i64> = v
.into_iter()
.take(MAX_HASH_HISTORY_LENGTH_STORED - 1)
.collect();
v.insert(0, p_hash.0);
v
};
let r: Result<Option<(Vec<i64>, serde_json::Value)>> = match o {
Some(clashing_script)
if clashing_script.path == ns.path
&& clashing_script.hash.0 != p_hash.0 =>
{
Err(Error::BadRequest(format!(
"Path conflict for {} with non-archived hash {}",
&ns.path, &clashing_script.hash
)))
}
Some(_) => Ok(Some((ph, ps.extra_perms))),
None => Ok(Some((ph, ps.extra_perms))),
};
sqlx::query!(
"UPDATE script SET archived = true WHERE hash = $1 AND workspace_id = $2",
p_hash.0,
&w_id
)
.execute(&mut tx)
.await?;
r
}
}?;
let p_hashes = parent_hashes_and_perms.as_ref().map(|v| &v.0[..]);
let extra_perms = parent_hashes_and_perms
.as_ref()
.map(|v| v.1.clone())
.unwrap_or(json!({}));
//::text::json is to ensure we use serde_json with preserve order
sqlx::query!(
"INSERT INTO script (workspace_id, hash, path, parent_hashes, summary, description, content, \
created_by, schema, is_template, extra_perms, lock) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::text::json, $10, $11, $12)",
&w_id,
&hash.0,
ns.path,
p_hashes,
ns.summary,
ns.description,
&ns.content,
&authed.username,
ns.schema.and_then(|x| serde_json::to_string(&x.0).ok()),
ns.is_template.unwrap_or(false),
extra_perms,
ns.lock.as_ref().map(|x| x.join("\n"))
)
.execute(&mut tx)
.await?;
let mut tx = if ns.lock.is_none() {
let dependencies = parser::parse_imports(&ns.content)?;
let (_, tx) = jobs::push(
tx,
&w_id,
jobs::JobPayload::Dependencies { hash, dependencies },
None,
&authed.username,
owner_to_token_owner(&authed.username, false),
None,
None,
None,
false,
)
.await?;
tx
} else {
tx
};
if p_hashes.is_some() && !p_hashes.unwrap().is_empty() {
audit_log(
&mut tx,
&authed.username,
"scripts.update",
ActionKind::Update,
&w_id,
Some(&ns.path),
Some(
[
("hash", hash.to_string().as_str()),
("token", &truncate_token(&token)),
]
.into(),
),
)
.await?;
} else {
audit_log(
&mut tx,
&authed.username,
"scripts.create",
ActionKind::Create,
&w_id,
Some(&ns.path),
Some(
[
("workspace", w_id.as_str()),
("hash", hash.to_string().as_str()),
("token", &truncate_token(&token)),
]
.into(),
),
)
.await?;
}
tx.commit().await?;
Ok((StatusCode::CREATED, format!("{}", hash)))
}
async fn get_script_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<Script> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let script_o = sqlx::query_as::<_, Script>(
"SELECT * FROM script WHERE path = $1 AND (workspace_id = $2 OR workspace_id = 'starter') AND
created_at = (SELECT max(created_at) FROM script WHERE path = $1 AND archived = false AND (workspace_id = $2 OR workspace_id = 'starter'))",
)
.bind(path)
.bind(w_id)
.fetch_optional(&mut tx)
.await?;
tx.commit().await?;
let script = crate::utils::not_found_if_none(script_o, "Script", path)?;
Ok(Json(script))
}
async fn get_script_by_hash_internal<'c>(
db: &mut Transaction<'c, Postgres>,
workspace_id: &str,
hash: &ScriptHash,
) -> Result<Script> {
let script_o = sqlx::query_as::<_, Script>(
"SELECT * FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
)
.bind(hash)
.bind(workspace_id)
.fetch_optional(db)
.await?;
let script = crate::utils::not_found_if_none(script_o, "Script", hash.to_string())?;
Ok(script)
}
async fn get_script_by_hash(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, hash)): Path<(String, ScriptHash)>,
) -> JsonResult<Script> {
let mut tx = user_db.begin(&authed).await?;
let r = get_script_by_hash_internal(&mut tx, &w_id, &hash).await?;
tx.commit().await?;
Ok(Json(r))
}
#[derive(FromRow, Serialize)]
struct DeploymentStatus {
lock: Option<String>,
lock_error_logs: Option<String>,
}
async fn get_deployment_status(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, hash)): Path<(String, ScriptHash)>,
) -> JsonResult<DeploymentStatus> {
let mut tx = user_db.begin(&authed).await?;
let status_o: Option<DeploymentStatus> = sqlx::query_as!(DeploymentStatus,
"SELECT lock, lock_error_logs FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')",
hash.0,
w_id,
)
.fetch_optional(&mut tx)
.await?;
let status = crate::utils::not_found_if_none(status_o, "DeploymentStatus", hash.to_string())?;
tx.commit().await?;
Ok(Json(status))
}
async fn archive_script_by_path(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<()> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let hash: i64 = sqlx::query_scalar!(
"UPDATE script SET archived = true WHERE path = $1 AND workspace_id = $2 RETURNING hash",
path,
&w_id
)
.fetch_one(&db)
.await?;
audit_log(
&mut tx,
&authed.username,
"scripts.archive",
ActionKind::Delete,
&w_id,
Some(&ScriptHash(hash).to_string()),
Some([("workspace", w_id.as_str())].into()),
)
.await?;
tx.commit().await?;
Ok(())
}
async fn archive_script_by_hash(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path((w_id, hash)): Path<(String, ScriptHash)>,
) -> JsonResult<Script> {
let mut tx = user_db.begin(&authed).await?;
let script = sqlx::query_as::<_, Script>(
"UPDATE script SET archived = true WHERE hash = $1 RETURNING *",
)
.bind(&hash.0)
.fetch_one(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"scripts.archive",
ActionKind::Delete,
&w_id,
Some(&hash.to_string()),
Some([("workspace", w_id.as_str())].into()),
)
.await?;
tx.commit().await?;
Ok(Json(script))
}
async fn delete_script_by_hash(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, hash)): Path<(String, ScriptHash)>,
) -> JsonResult<Script> {
let mut tx = user_db.begin(&authed).await?;
require_admin(authed.is_admin, &authed.username)?;
let script = sqlx::query_as::<_, Script>(
"UPDATE script SET content = '', archived = true, deleted = true WHERE hash = $1 AND workspace_id = $2\
RETURNING *",
)
.bind(&hash.0)
.bind(&w_id)
.fetch_one(&db)
.await?;
audit_log(
&mut tx,
&authed.username,
"scripts.delete",
ActionKind::Delete,
&w_id,
Some(&hash.to_string()),
Some([("workspace", w_id.as_str())].into()),
)
.await?;
tx.commit().await?;
Ok(Json(script))
}
async fn parse_code_to_jsonschema(
Json(code): Json<String>,
) -> JsonResult<parser::MainArgSignature> {
parser::parse_signature(&code).map(Json)
}
pub fn to_i64(s: &str) -> Result<i64> {
let v = hex::decode(s)?;
let nb: u64 = u64::from_be_bytes(
v[0..8]
.try_into()
.map_err(|_| hex::FromHexError::InvalidStringLength)?,
);
Ok(nb as i64)
}
pub fn to_hex_string(i: &i64) -> String {
hex::encode(i.to_be_bytes())
}

56
backend/src/static_assets.rs Normal file
View File

@@ -0,0 +1,56 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use axum::{
body::{self, BoxBody},
http::{header, Response, Uri},
response::IntoResponse,
};
use rust_embed::RustEmbed;
// static_handler is a handler that serves static files from the
pub async fn static_handler(uri: Uri) -> impl IntoResponse {
let path = uri.path().trim_start_matches('/').to_string();
StaticFile(path)
}
#[derive(RustEmbed)]
#[folder = "../frontend/build/"]
struct Asset;
pub struct StaticFile<T>(pub T);
impl<T> IntoResponse for StaticFile<T>
where
T: Into<String>,
{
fn into_response(self) -> Response<BoxBody> {
let path = self.0.into();
serve_path(path)
}
}
fn serve_path(path: String) -> Response<BoxBody> {
if path.starts_with("api/") {
return Response::builder()
.status(404)
.body(body::boxed(body::Empty::new()))
.unwrap();
}
match Asset::get(path.as_str()) {
Some(content) => {
let body = body::boxed(body::Full::from(content.data));
let mime = mime_guess::from_path(path).first_or_octet_stream();
Response::builder()
.header(header::CONTENT_TYPE, mime.as_ref())
.header(header::CACHE_CONTROL, "max-age=3600".to_owned())
.body(body)
.unwrap()
}
None => serve_path("200.html".to_owned()),
}
}

1411
backend/src/users.rs Normal file
View File

File diff suppressed because it is too large Load Diff

93
backend/src/utils.rs Normal file
View File

@@ -0,0 +1,93 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use rand::{distributions::Alphanumeric, thread_rng, Rng};
use serde::Deserialize;
use sqlx::{Postgres, Transaction};
use crate::error::{Error, Result};
pub const MAX_PER_PAGE: usize = 1000;
pub const DEFAULT_PER_PAGE: usize = 30;
#[derive(Deserialize)]
pub struct Pagination {
pub page: Option<usize>,
pub per_page: Option<usize>,
}
#[derive(Deserialize)]
pub struct StripPath(String);
impl StripPath {
pub fn to_path(&self) -> &str {
self.0.strip_prefix('/').unwrap()
}
}
pub async fn require_super_admin<'c>(
db: &mut Transaction<'c, Postgres>,
email: Option<String>,
) -> Result<()> {
let is_admin = sqlx::query_scalar!(
"SELECT super_admin FROM password WHERE email = $1",
email.as_ref()
)
.fetch_one(db)
.await?;
if !is_admin {
Err(Error::NotAuthorized(
"This endpoint require caller to be a super admin".to_owned(),
))
} else {
Ok(())
}
}
pub fn require_admin(is_admin: bool, username: &str) -> Result<()> {
if !is_admin {
Err(Error::NotAuthorized(format!(
"This endpoint require caller {} to be an admin",
username
)))
} else {
Ok(())
}
}
pub fn rd_string(len: usize) -> String {
thread_rng()
.sample_iter(&Alphanumeric)
.take(len)
.map(char::from)
.collect()
}
pub fn paginate(pagination: Pagination) -> (usize, usize) {
let per_page = pagination
.per_page
.unwrap_or(DEFAULT_PER_PAGE)
.max(1)
.min(MAX_PER_PAGE);
let offset = (pagination.page.unwrap_or(1).max(1) - 1) * per_page;
(per_page, offset)
}
pub fn not_found_if_none<T, U: AsRef<str>>(opt: Option<T>, kind: &str, name: U) -> Result<T> {
if let Some(o) = opt {
Ok(o)
} else {
Err(Error::NotFound(format!(
"{} not found at name {}",
kind,
name.as_ref()
)))
}
}
pub fn get_owner_from_path(path: &str) -> String {
path.split('/').take(2).collect::<Vec<_>>().join("/")
}

365
backend/src/variables.rs Normal file
View File

@@ -0,0 +1,365 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
audit::{audit_log, ActionKind},
db::{UserDB, DB},
error::{Error, JsonResult, Result},
users::Authed,
utils::StripPath,
};
use axum::{
extract::{Extension, Path, Query},
routing::{delete, get, post},
Json, Router,
};
use hyper::StatusCode;
use magic_crypt::{MagicCrypt256, MagicCryptTrait};
use serde::{Deserialize, Serialize};
use sqlx::{FromRow, Postgres, Transaction};
pub fn workspaced_service() -> Router {
Router::new()
.route("/list", get(list_variables))
.route("/list_contextual", get(list_contextual_variables))
.route("/get/*path", get(get_variable))
.route("/update/*path", post(update_variable))
.route("/delete/*path", delete(delete_variable))
.route("/create", post(create_variable))
}
#[derive(Serialize, Clone)]
pub struct ContextualVariable {
pub name: String,
pub value: String,
pub description: String,
}
#[derive(Serialize, Deserialize, FromRow)]
pub struct ListableVariable {
pub workspace_id: String,
pub path: String,
pub value: Option<String>,
pub is_secret: bool,
pub description: String,
pub extra_perms: serde_json::Value,
}
#[derive(Deserialize)]
pub struct CreateVariable {
pub path: String,
pub value: String,
pub is_secret: bool,
pub description: String,
}
#[derive(Deserialize)]
struct EditVariable {
path: Option<String>,
value: Option<String>,
is_secret: Option<bool>,
description: Option<String>,
}
pub fn get_reserved_variables(
w_id: &str,
token: &str,
email: &str,
username: &str,
job_id: &str,
) -> [ContextualVariable; 5] {
[
ContextualVariable {
name: "WM_WORKSPACE".to_string(),
value: w_id.to_string(),
description: "Workspace id of the current script".to_string()
},
ContextualVariable {
name: "WM_TOKEN".to_string(),
value: token.to_string(),
description: "Token ephemeral to the current script with equal permission to the permission of the run (Usable as a bearer token)".to_string()
},
ContextualVariable {
name: "WM_EMAIL".to_string(),
value: email.to_string(),
description: "Email of the user that executed the current script".to_string()
},
ContextualVariable {
name: "WM_USERNAME".to_string(),
value: username.to_string(),
description: "Username of the user that executed the current script".to_string()
},
ContextualVariable {
name: "WM_JOB_ID".to_string(),
value: job_id.to_string(),
description: "Job id of the current script".to_string()
},
]
}
async fn list_contextual_variables(
Path(w_id): Path<String>,
Authed {
username, email, ..
}: Authed,
) -> JsonResult<Vec<ContextualVariable>> {
Ok(Json(
get_reserved_variables(
&w_id,
"q1A0qcPuO00yxioll7iph76N9CJDqn",
&email.unwrap_or_else(|| "no email".to_string()),
&username,
"017e0ad5-f499-73b6-5488-92a61c5196dd",
)
.to_vec(),
))
}
async fn list_variables(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<ListableVariable>> {
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as::<_, ListableVariable>(
"SELECT workspace_id, path, CASE WHEN is_secret IS TRUE THEN null ELSE value::text END as value, is_secret, description, extra_perms from variable
WHERE (workspace_id = $1 OR (is_secret IS NOT TRUE AND workspace_id = 'starter')) ORDER BY path",
)
.bind(&w_id)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
}
#[derive(Deserialize)]
struct GetVariableQuery {
decrypt_secret: Option<bool>,
}
async fn get_variable(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Query(q): Query<GetVariableQuery>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> JsonResult<ListableVariable> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let variable_o = sqlx::query_as::<_, ListableVariable>(
"SELECT * from variable WHERE path = $1 AND (workspace_id = $2 OR (is_secret IS NOT TRUE AND workspace_id = 'starter'))",
)
.bind(&path)
.bind(&w_id)
.fetch_optional(&mut tx)
.await?;
let variable = crate::utils::not_found_if_none(variable_o, "Variable", &path)?;
let decrypt_secret = q.decrypt_secret.unwrap_or(true);
let r = if variable.is_secret {
audit_log(
&mut tx,
&authed.username,
"variables.decrypt_secret",
ActionKind::Execute,
&w_id,
Some(&variable.path),
None,
)
.await?;
let value = variable.value.unwrap_or_else(|| "".to_string());
ListableVariable {
value: if !value.is_empty() && decrypt_secret {
let mc = build_crypt(&mut tx, &w_id).await?;
Some(
mc.decrypt_base64_to_string(value)
.map_err(|e| Error::InternalErr(e.to_string()))?,
)
} else {
None
},
..variable
}
} else {
variable
};
tx.commit().await?;
Ok(Json(r))
}
async fn create_variable(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
Json(variable): Json<CreateVariable>,
) -> Result<(StatusCode, String)> {
let mut tx = user_db.begin(&authed).await?;
let value = if variable.is_secret {
let mc = build_crypt(&mut tx, &w_id).await?;
encrypt(&mc, variable.value)
} else {
variable.value
};
sqlx::query!(
"INSERT INTO variable
(workspace_id, path, value, is_secret, description)
VALUES ($1, $2, $3, $4, $5)",
&w_id,
variable.path,
value,
variable.is_secret,
variable.description
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"variables.create",
ActionKind::Create,
&w_id,
Some(&variable.path),
None,
)
.await?;
tx.commit().await?;
Ok((
StatusCode::CREATED,
format!("variable {} created", variable.path),
))
}
async fn delete_variable(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
) -> Result<String> {
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
sqlx::query!(
"DELETE FROM variable WHERE path = $1 AND workspace_id = $2",
path,
w_id
)
.execute(&db)
.await?;
audit_log(
&mut tx,
&authed.username,
"variables.delete",
ActionKind::Delete,
&w_id,
Some(path),
None,
)
.await?;
tx.commit().await?;
Ok(format!("variable {} deleted", path))
}
async fn update_variable(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Extension(db): Extension<DB>,
Path((w_id, path)): Path<(String, StripPath)>,
Json(ns): Json<EditVariable>,
) -> Result<String> {
use sql_builder::prelude::*;
let path = path.to_path();
let mut tx = user_db.begin(&authed).await?;
let mut sqlb = SqlBuilder::update_table("variable");
sqlb.and_where_eq("path", "?".bind(&path));
sqlb.and_where_eq("workspace_id", "?".bind(&w_id));
if let Some(npath) = &ns.path {
sqlb.set_str("path", npath);
}
if let Some(nvalue) = ns.value {
let is_secret = sqlx::query_scalar!(
"SELECT is_secret from variable WHERE path = $1 AND workspace_id = $2",
&path,
&w_id
)
.fetch_optional(&mut tx)
.await?
.unwrap_or(false);
let value = if is_secret {
let mc = build_crypt(&mut tx, &w_id).await?;
encrypt(&mc, nvalue)
} else {
nvalue
};
sqlb.set_str("value", &value);
}
if let Some(desc) = ns.description {
sqlb.set_str("description", &desc);
}
if let Some(nbool) = ns.is_secret {
if !nbool {
return Err(Error::BadRequest(
"A variable can not be updated to be non secret".to_owned(),
));
}
sqlb.set_str("is_secret", nbool);
}
let sql = sqlb.sql().map_err(|e| Error::InternalErr(e.to_string()))?;
sqlx::query(&sql).execute(&db).await?;
audit_log(
&mut tx,
&authed.username,
"variables.update",
ActionKind::Update,
&w_id,
Some(path),
None,
)
.await?;
tx.commit().await?;
Ok(format!("variable {} updated (npath: {:?})", path, ns.path))
}
pub async fn build_crypt<'c>(
db: &mut Transaction<'c, Postgres>,
w_id: &str,
) -> Result<MagicCrypt256> {
let key = sqlx::query_scalar!(
"SELECT key FROM workspace_key WHERE workspace_id = $1 AND kind = 'cloud'",
w_id
)
.fetch_one(db)
.await?;
Ok(magic_crypt::new_magic_crypt!(key, 256))
}
pub fn encrypt(mc: &MagicCrypt256, value: String) -> String {
mc.encrypt_str_to_base64(value)
}

726
backend/src/worker.rs Normal file
View File

@@ -0,0 +1,726 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use itertools::Itertools;
use std::{
process::{ExitStatus, Stdio},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
time::Duration,
};
use crate::{
db::DB,
error::Error,
jobs::{
add_completed_job, add_completed_job_error, handle_flow, postprocess_queued_job, pull,
update_flow_status_after_job_completion, update_flow_status_in_progress, JobKind,
QueuedJob,
},
parser::{self, Typ},
scripts::ScriptHash,
users::{create_token_for_owner, get_email_from_username},
variables,
};
use serde_json::{json, Map, Value};
use tokio::{
fs::{DirBuilder, File},
io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader},
process::{Child, Command},
sync::Mutex,
time::Instant,
};
use async_recursion::async_recursion;
use tokio::sync::mpsc;
const TMP_DIR: &str = "/tmp/windmill";
const PIP_CACHE_DIR: &str = "/tmp/windmill/cache/pip";
const NUM_SECS_ENV_CHECK: u64 = 15;
const INCLUDE_DEPS_SH_CONTENT: &str = include_str!("../../nsjail/download_deps.sh");
const NSJAIL_CONFIG_DOWNLOAD_CONTENT: &str = include_str!("../../nsjail/download.config.proto");
const NSJAIL_CONFIG_RUN_CONTENT: &str = include_str!("../../nsjail/run.config.proto");
pub async fn run_worker(
db: &DB,
timeout: i32,
worker_instance: &str,
worker_name: String,
i_worker: u64,
num_workers: u64,
_mutex: Arc<Mutex<i32>>,
ip: &str,
sleep_queue: u64,
base_url: &str,
tx: tokio::sync::broadcast::Sender<()>,
) {
let worker_dir = format!("{TMP_DIR}/{worker_name}");
tracing::debug!(worker_dir = %worker_dir, worker_name = %worker_name, "Creating worker dir");
DirBuilder::new()
.recursive(true)
.create(&worker_dir)
.await
.expect("could not create initial worker dir");
DirBuilder::new()
.recursive(true)
.create(&PIP_CACHE_DIR)
.await
.expect("could not create initial worker dir");
let _ = write_file(&worker_dir, "download_deps.sh", INCLUDE_DEPS_SH_CONTENT).await;
let mut last_ping = Instant::now() - Duration::from_secs(NUM_SECS_ENV_CHECK + 1);
insert_initial_ping(worker_instance, &worker_name, ip, db).await;
let mut jobs_executed = 0;
let mut rx = tx.subscribe();
loop {
if last_ping.elapsed().as_secs() > NUM_SECS_ENV_CHECK {
sqlx::query!(
"UPDATE worker_ping SET ping_at = $1, jobs_executed = $2 WHERE worker = $3",
chrono::Utc::now(),
jobs_executed,
&worker_name
)
.execute(db)
.await
.expect("update worker ping");
last_ping = Instant::now();
}
match pull(db).await {
Ok(Some(job)) => {
jobs_executed += 1;
tracing::info!(worker = %worker_name, id = %job.id, "Fetched job");
let job2 = job.clone();
if let Some(err) =
handle_queued_job(job, db, timeout, &worker_name, &worker_dir, base_url)
.await
.err()
{
let err_string = err.to_string().clone();
let _ = add_completed_job_error(
db,
&job2,
"Unexpected error during job execution:\n".to_string(),
err,
)
.await;
let _ =
postprocess_queued_job(job2.schedule_path, &job2.workspace_id, job2.id, db)
.await;
tracing::error!(job_id = %job2.id, "Error handling job: {err_string}");
};
}
Ok(None) => (),
Err(err) => {
tracing::error!(worker = %worker_name, "run_worker: pulling jobs: {}", err);
}
};
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(sleep_queue * num_workers)) => (),
_ = rx.recv() => {
println!("received killpill for worker {}", i_worker);
break;
}
}
}
}
async fn insert_initial_ping(worker_instance: &str, worker_name: &str, ip: &str, db: &DB) {
sqlx::query!(
"INSERT INTO worker_ping (worker_instance, worker, ip) VALUES ($1, $2, $3)",
worker_instance,
worker_name,
ip
)
.execute(db)
.await
.expect("insert worker_ping initial value");
}
async fn handle_queued_job(
job: QueuedJob,
db: &sqlx::Pool<sqlx::Postgres>,
timeout: i32,
worker_name: &str,
worker_dir: &str,
base_url: &str,
) -> crate::error::Result<()> {
let job_id = job.id;
let w_id = &job.workspace_id.clone();
match job.job_kind {
JobKind::FlowPreview | JobKind::Flow => {
let args = match &job.args {
Some(serde_json::Value::Object(m)) => Some(m.to_owned()),
_ => None,
};
handle_flow(&job, db, args).await?;
}
_ => {
let mut logs = "".to_string();
let mut last_line = "{}".to_string();
if job.is_flow_step {
update_flow_status_in_progress(
db,
&job.workspace_id,
job.parent_job
.ok_or_else(|| Error::InternalErr(format!("expected parent job")))?,
job.id,
)
.await?;
}
let execution = handle_job(
&job,
db,
timeout,
worker_name,
worker_dir,
&mut logs,
&mut last_line,
base_url,
)
.await;
match execution {
Ok(r) => {
add_completed_job(db, &job, true, r.result.clone(), logs).await?;
if job.is_flow_step {
update_flow_status_after_job_completion(db, &job, true, r.result).await?;
}
}
Err(e) => {
let (_, output_map) = add_completed_job_error(db, &job, logs, e).await?;
if job.is_flow_step {
update_flow_status_after_job_completion(db, &job, false, Some(output_map))
.await?;
}
}
};
let _ = postprocess_queued_job(job.schedule_path, &w_id, job_id, db).await;
}
}
Ok(())
}
struct JobResult {
result: Option<Map<String, Value>>,
}
async fn write_file(dir: &str, path: &str, content: &str) -> Result<File, Error> {
let path = format!("{}/{}", dir, path);
let mut file = File::create(&path).await?;
file.write_all(content.as_bytes()).await?;
Ok(file)
}
#[async_recursion]
async fn transform_json_value(token: &str, workspace: &str, base_url: &str, v: Value) -> Value {
match v {
Value::String(y) if y.starts_with("$var:") => {
let path = y.strip_prefix("$var:").unwrap();
let v = crate::client::get_variable(workspace, path, token, base_url)
.await
.unwrap_or_else(|_| format!("error fetching variable {path}"));
Value::String(v)
}
Value::String(y) if y.starts_with("$res:") => {
let path = y.strip_prefix("$res:").unwrap();
let v = crate::client::get_resource(workspace, path, token, base_url)
.await
.ok()
.flatten()
.unwrap_or_else(|| Value::String(format!("error fetching resource {path}")));
transform_json_value(token, workspace, base_url, v).await
}
Value::Object(mut m) => {
for (a, b) in m.clone().into_iter() {
m.insert(a, transform_json_value(token, workspace, base_url, b).await);
}
Value::Object(m)
}
a @ _ => a,
}
}
#[allow(clippy::too_many_arguments)]
async fn handle_job(
job: &QueuedJob,
db: &DB,
timeout: i32,
worker_name: &str,
worker_dir: &str,
mut logs: &mut String,
mut last_line: &mut String,
base_url: &str,
) -> Result<JobResult, Error> {
tracing::info!(
worker = %worker_name,
job_id = %job.id,
"handling job"
);
logs.push_str(&format!("job {} on worker {}\n", &job.id, &worker_name));
let job_dir = format!("{worker_dir}/{}", job.id);
DirBuilder::new()
.recursive(true)
.create(&format!("{job_dir}/dependencies"))
.await
.expect("could not create initial job dir");
let mut status: Result<ExitStatus, Error>;
if matches!(job.job_kind, JobKind::Dependencies) {
let requirements = job
.raw_code
.as_ref()
.ok_or_else(|| Error::ExecutionErr("missing requirements".to_string()))?;
logs.push_str(&format!("content of requirements:\n{}\n", &requirements));
let file = "requirements.in";
write_file(&job_dir, file, &requirements).await?;
let child = Command::new("pip-compile")
.current_dir(&job_dir)
.args(vec!["-q", "--no-header", file])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
if status.is_ok() && status.as_ref().unwrap().success() {
let path_lock = format!("{}/requirements.txt", job_dir);
let mut file = File::open(path_lock).await?;
let mut content = "".to_string();
file.read_to_string(&mut content).await?;
content = content
.lines()
.filter(|x| !x.trim_start().starts_with('#'))
.map(|x| x.to_string())
.collect::<Vec<String>>()
.join("\n");
let as_json = json!(content);
*last_line =
format!(r#"{{ "success": "Successful lock file generation", "lock": {as_json} }}"#);
sqlx::query!(
"UPDATE script SET lock = $1 WHERE hash = $2 AND workspace_id = $3",
&content,
&job.script_hash.unwrap_or(ScriptHash(0)).0,
&job.workspace_id
)
.execute(db)
.await?;
} else {
sqlx::query!(
"UPDATE script SET lock_error_logs = $1 WHERE hash = $2 AND workspace_id = $3",
&logs.clone(),
&job.script_hash.unwrap_or(ScriptHash(0)).0,
&job.workspace_id
)
.execute(db)
.await?;
}
} else {
let (inner_content, requirements_o) = if matches!(job.job_kind, JobKind::Preview) {
let code = (job.raw_code.as_ref().unwrap_or(&"no raw code".to_owned())).to_owned();
let reqs = parser::parse_imports(&code)?.join("\n");
(code, Some(reqs))
} else {
sqlx::query_as::<_, (String, Option<String>)>("SELECT content, lock FROM script WHERE hash = $1 AND (workspace_id = $2 OR workspace_id = 'starter')")
.bind(&job.script_hash.unwrap_or(ScriptHash(0)).0)
.bind(&job.workspace_id)
.fetch_optional(db)
.await?
.ok_or_else(|| Error::InternalErr(format!("expected content and lock")))?
};
let requirements =
requirements_o.ok_or_else(|| Error::InternalErr(format!("lockfile missing")))?;
let _ = write_file(
&job_dir,
"download.config.proto",
&NSJAIL_CONFIG_DOWNLOAD_CONTENT
.replace("{JOB_DIR}", &job_dir)
.replace("{WORKER_DIR}", &worker_dir)
.replace("{CACHE_DIR}", PIP_CACHE_DIR),
)
.await?;
let _ = write_file(&job_dir, "requirements.txt", &requirements).await?;
let child = Command::new("nsjail")
.current_dir(&job_dir)
.args(vec!["--config", "download.config.proto"])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
logs.push_str("\n--- DEPENDENCIES INSTALL ---\n");
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
if status.is_ok() {
logs.push_str("\n\n--- CODE EXECUTION ---\n");
set_logs(logs, job.id, db).await;
let _ = write_file(&job_dir, "inner.py", &inner_content).await?;
let sig = crate::parser::parse_signature(&inner_content)?;
let transforms = sig.args.into_iter().map(|x| match x.typ {
Typ::Bytes => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n kwargs[\"{}\"] = base64.b64decode(kwargs[\"{}\"])\n", x.name, x.name, x.name, x.name),
Typ::Datetime => format!("if \"{}\" in kwargs and kwargs[\"{}\"] is not None:\n kwargs[\"{}\"] = datetime.strptime(kwargs[\"{}\"], '%Y-%m-%dT%H:%M')\n", x.name, x.name, x.name, x.name),
_ => "".to_string()
}).collect::<Vec<String>>().join("");
let tx = db.begin().await?;
let token = create_token_for_owner(
&db,
&job.workspace_id,
&job.permissioned_as,
crate::users::NewToken {
label: Some("ephemeral-script".to_string()),
expiration: Some(
chrono::Utc::now() + chrono::Duration::seconds((timeout * 2).into()),
),
},
&job.created_by,
)
.await?;
let args = if let Some(args) = &job.args {
Some(transform_json_value(&token, &job.workspace_id, base_url, args.clone()).await)
} else {
None
};
let ser_args = serde_json::to_string(&args)
.map_err(|e| Error::ExecutionErr(e.to_string()))?
.replace("\\\"", "\\\\\"");
let wrapper_content: String = format!(
r#"
import json
import base64
from datetime import datetime
inner_script = __import__("inner")
kwargs = json.loads("""{ser_args}""", strict=False)
for k, v in kwargs.items():
if v == '<function call>':
kwargs[k] = None
{transforms}
res = inner_script.main(**kwargs)
if res is None:
res = {{}}
if isinstance(res, tuple):
res = {{f"res{{i+1}}": v for i, v in enumerate(res)}}
if not isinstance(res, dict):
res = {{ "res1": res }}
res_json = json.dumps(res, separators=(',', ':'), default=str).replace('\n', '')
print()
print("result:")
print(res_json)
"#,
);
write_file(&job_dir, "main.py", &wrapper_content).await?;
tx.commit().await?;
let reserved_variables = variables::get_reserved_variables(
&job.workspace_id,
&token,
&get_email_from_username(&job.created_by, db)
.await?
.unwrap_or_else(|| "nosuitable@email.xyz".to_string()),
&job.created_by,
&job.id.to_string(),
)
.into_iter()
.map(|rv| (rv.name, rv.value));
let _ = write_file(
&job_dir,
"run.config.proto",
&NSJAIL_CONFIG_RUN_CONTENT.replace("{JOB_DIR}", &job_dir),
)
.await?;
let child = Command::new("nsjail")
.current_dir(&job_dir)
.envs(reserved_variables)
.args(vec![
"--config",
"run.config.proto",
"--",
"/usr/local/bin/python3",
"-u",
"/tmp/main.py",
])
.stdout(Stdio::piped())
.stderr(Stdio::piped())
.spawn()?;
status = handle_child(job, db, &mut logs, &mut last_line, timeout, child).await;
}
}
tokio::fs::remove_dir_all(job_dir).await?;
if status.is_ok() && status.as_ref().unwrap().success() {
let result = serde_json::from_str::<Map<String, Value>>(last_line).map_err(|e| {
Error::ExecutionErr(format!(
"result {} is not parsable.\n err: {}",
last_line,
e.to_string()
))
})?;
Ok(JobResult {
result: Some(result),
})
} else {
let err = match status {
Ok(_) => {
let s = format!(
"Error during execution of the script\nlast 5 logs lines:\n{}",
logs.lines()
.skip(logs.lines().count().max(5) - 5)
.join("\n")
);
logs.push_str("\n\n--- ERROR ---\n");
s
}
Err(err) => format!("error before termination: {err}"),
};
Err(Error::ExecutionErr(err))
}
}
async fn handle_child(
job: &QueuedJob,
db: &DB,
logs: &mut String,
last_line: &mut String,
timeout: i32,
mut child: Child,
) -> crate::error::Result<ExitStatus> {
let stderr = child
.stderr
.take()
.expect("child did not have a handle to stdout");
let stdout = child
.stdout
.take()
.expect("child did not have a handle to stdout");
let mut reader = BufReader::new(stdout).lines();
let mut stderr_reader = BufReader::new(stderr).lines();
let done = Arc::new(AtomicBool::new(false));
let done2 = done.clone();
let done3 = done.clone();
// Ensure the child process is spawned in the runtime so it can
// make progress on its own while we await for any output.
let handle = tokio::spawn(async move {
let inner_done = done2.clone();
let r: Result<ExitStatus, anyhow::Error> = tokio::select! {
r = child.wait() => {
inner_done.store(true, Ordering::Relaxed);
Ok(r?)
}
_ = async move {
loop {
if done2.load(Ordering::Relaxed) {
break;
}
tokio::time::sleep(Duration::from_secs(5)).await;
}
} => {
child.kill().await?;
return Err(Error::ExecutionErr("execution interrupted (likely timeout or cancel)".to_string()).into())
}
};
r
});
let (tx, mut rx) = mpsc::channel::<String>(100);
let id = job.id;
tokio::spawn(async move {
loop {
let send = tokio::select! {
Ok(Some(out)) = reader.next_line() => {
tx.send(out).await
},
Ok(Some(err)) = stderr_reader.next_line() => tx.send(err).await,
else => {
break
},
};
if send.err().is_some() {
tracing::error!("error sending log line");
};
}
});
let db2 = db.clone();
tokio::spawn(async move {
while !&done3.load(Ordering::Relaxed) {
let q = sqlx::query!(
"UPDATE queue SET last_ping = $1 WHERE id = $2",
chrono::Utc::now(),
id
)
.execute(&db2)
.await;
if q.is_err() {
tracing::error!("error setting last ping for id {}", id);
}
tokio::time::sleep(Duration::from_secs(5)).await;
}
});
let mut start = logs.chars().count();
loop {
tokio::select! {
_ = tokio::time::sleep(Duration::from_millis(500)) => {
let end = logs.chars().count();
let to_send = logs.chars().skip(start).collect::<String>();
if start != end {
concat_logs(&to_send, id, db).await;
start = end;
}
let canceled = sqlx::query_scalar!("SELECT canceled FROM queue WHERE id = $1", id)
.fetch_one(db)
.await
.map_err(|_| tracing::error!("error getting canceled for id {}", id));
if canceled.unwrap_or(false) {
tracing::info!("killed after cancel: {}", job.id);
done.store(true, Ordering::Relaxed);
}
let has_timeout = job
.started_at
.map(|sa| (chrono::Utc::now() - sa).num_seconds() > timeout as i64)
.unwrap_or(false);
if has_timeout {
let q = sqlx::query(&format!(
"UPDATE queue SET canceled = true, canceled_by = 'timeout', \
canceled_reason = 'duration > {}' WHERE id = $1",
timeout
))
.bind(id)
.execute(db)
.await;
if q.is_err() {
tracing::error!("error setting canceled for id {}", id);
}
}
},
nl = rx.recv() => {
if let Some(nl) = nl {
logs.push('\n');
logs.push_str(&nl);
*last_line = nl;
} else {
let to_send = logs.chars().skip(start).collect::<String>();
concat_logs(&to_send, id, db).await;
break;
}
},
}
}
let status = handle
.await
.map_err(|e| Error::ExecutionErr(e.to_string()))??;
Ok(status)
}
async fn set_logs(logs: &str, id: uuid::Uuid, db: &DB) {
if sqlx::query!(
"UPDATE queue SET logs = $1 WHERE id = $2",
logs.to_owned(),
id
)
.execute(db)
.await
.is_err()
{
tracing::error!("error updating logs for id {}", id)
};
}
async fn concat_logs(logs: &str, id: uuid::Uuid, db: &DB) {
if sqlx::query!(
"UPDATE queue SET logs = concat(logs, $1::text) WHERE id = $2",
logs.to_owned(),
id
)
.execute(db)
.await
.is_err()
{
tracing::error!("error updating logs for id {}", id)
};
}
pub async fn restart_zombie_jobs_periodically(
db: &DB,
timeout: i32,
mut rx: tokio::sync::broadcast::Receiver<()>,
) {
loop {
let restarted = sqlx::query_scalar!(
"UPDATE queue SET running = false WHERE last_ping < $1 RETURNING id",
chrono::Utc::now() - chrono::Duration::seconds(timeout as i64 * 2)
)
.fetch_all(db)
.await
.ok()
.unwrap_or_else(|| vec![]);
if restarted.len() > 0 {
tracing::info!("restarted zombie jobs {restarted:?}");
}
tokio::select! {
_ = tokio::time::sleep(Duration::from_secs(60)) => (),
_ = rx.recv() => {
println!("received killpill for monitor job");
break;
}
}
}
}

51
backend/src/worker_ping.rs Normal file
View File

@@ -0,0 +1,51 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{db::UserDB, error::JsonResult, users::Authed, utils::Pagination};
use axum::{
extract::{Extension, Query},
routing::get,
Json, Router,
};
use serde::{Deserialize, Serialize};
use sqlx::FromRow;
pub fn global_service() -> Router {
Router::new().route("/list", get(list_worker_pings))
}
#[derive(FromRow, Serialize, Deserialize)]
struct WorkerPing {
worker: String,
worker_instance: String,
ping_at: chrono::DateTime<chrono::Utc>,
started_at: chrono::DateTime<chrono::Utc>,
ip: String,
jobs_executed: i32,
}
async fn list_worker_pings(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Query(pagination): Query<Pagination>,
) -> JsonResult<Vec<WorkerPing>> {
let mut tx = user_db.begin(&authed).await?;
let (per_page, offset) = crate::utils::paginate(pagination);
let rows = sqlx::query_as!(
WorkerPing,
"SELECT * FROM worker_ping ORDER BY ping_at desc LIMIT $1 OFFSET $2",
per_page as i64,
offset as i64
)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
}

666
backend/src/workspaces.rs Normal file
View File

@@ -0,0 +1,666 @@
/*
* Author & Copyright: Ruben Fiszel 2021
* This file and its contents are licensed under the AGPL License.
* Please see the included NOTICE for copyright information and
* LICENSE-AGPL for a copy of the license.
*/
use crate::{
db::{UserDB, DB},
error::{Error, JsonResult, Result},
users::{Authed, WorkspaceInvite}, utils::{require_admin, require_super_admin, Pagination}, audit::{audit_log, ActionKind}, scripts::{Script, Schema}, resources::{Resource, ResourceType}, flow::Flow, variables::ListableVariable,
};
use axum::{extract::{Extension, Path, Query}, routing::{get, post, delete}, Json, Router, response::{IntoResponse}, body::StreamBody};
use hyper::{StatusCode, header};
use serde::{Deserialize, Serialize};
use sqlx::{FromRow};
use tempfile::TempDir;
use tokio::fs::File;
use tokio_util::io::ReaderStream;
pub fn workspaced_service() -> Router {
Router::new()
.route("/list_pending_invites", get(list_pending_invites))
.route("/update", post(edit_workspace))
.route("/delete", delete(delete_workspace))
.route("/invite_user", post(invite_user))
.route("/delete_invite", post(delete_invite))
.route("/get_settings", get(get_settings))
.route("/edit_slack_command", post(edit_slack_command))
.route("/tarball", get(tarball_workspace))
}
pub fn global_service() -> Router {
Router::new()
.route("/list_as_superadmin", get(list_workspaces_as_super_admin))
.route("/list", get(list_workspaces))
.route("/users", get(user_workspaces))
.route("/create", post(create_workspace))
.route("/exists", post(exists_workspace))
.route("/validate_username", post(validate_username))
.route("/validate_id", post(validate_id))
}
#[derive(FromRow, Serialize)]
struct Workspace {
id: String,
name: String,
owner: String,
domain: Option<String>,
deleted: bool
}
#[derive(FromRow, Serialize, Debug)]
pub struct WorkspaceSettings {
pub workspace_id: String,
pub slack_team_id: Option<String>,
pub slack_name: Option<String>,
pub slack_command_script: Option<String>
}
#[derive(sqlx::Type, Serialize, Deserialize, Debug)]
#[sqlx(type_name = "WORKSPACE_KEY_KIND", rename_all = "lowercase")]
pub enum WorkspaceKeyKind {
Cloud
}
#[derive(Deserialize)]
struct EditCommandScript {
slack_command_script: Option<String>
}
#[derive(Deserialize)]
struct CreateWorkspace {
id: String,
name: String,
username: String,
domain: Option<String>,
}
#[derive(Deserialize)]
struct EditWorkspace {
name: String,
owner: String,
domain: Option<String>
}
#[derive(Serialize)]
struct WorkspaceList {
pub email: String,
pub workspaces: Vec<UserWorkspace>,
}
#[derive(Serialize)]
struct UserWorkspace {
pub id: String,
pub name: String,
pub username: String,
}
#[derive(Deserialize)]
struct WorkspaceId {
pub id: String,
}
#[derive(Deserialize)]
struct ValidateUsername {
pub id: String,
pub username: String,
}
#[derive(Deserialize)]
pub struct NewWorkspaceInvite {
pub email: String,
pub is_admin: bool,
}
async fn list_pending_invites(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Path(w_id): Path<String>,
) -> JsonResult<Vec<WorkspaceInvite>> {
require_admin(authed.is_admin, &authed.username)?;
let mut tx = user_db.begin(&authed).await?;
let rows = sqlx::query_as!(WorkspaceInvite, "SELECT * from workspace_invite WHERE workspace_id = $1", w_id)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(rows))
}
async fn exists_workspace(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Json(WorkspaceId { id }): Json<WorkspaceId>
) -> JsonResult<bool> {
let mut tx = user_db.begin(&authed).await?;
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM workspace WHERE workspace.id = $1)",
id)
.fetch_one(&mut tx)
.await?
.unwrap_or(false);
tx.commit().await?;
Ok(Json(exists))
}
async fn list_workspaces(
authed: Authed,
Extension(user_db): Extension<UserDB>,
) -> JsonResult<Vec<Workspace>> {
let mut tx = user_db.begin(&authed).await?;
let workspaces = sqlx::query_as!(
Workspace,
"SELECT workspace.* FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = false",
authed.email.as_ref())
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(workspaces))
}
async fn get_settings(
authed: Authed,
Path(w_id): Path<String>,
Extension(user_db): Extension<UserDB>,
) -> JsonResult<WorkspaceSettings> {
let mut tx = user_db.begin(&authed).await?;
let settings = sqlx::query_as!(
WorkspaceSettings,
"SELECT * FROM workspace_settings WHERE workspace_id = $1",
&w_id)
.fetch_one(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(settings))
}
async fn edit_slack_command(
authed: Authed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Authed { is_admin, username, .. }: Authed,
Json(es): Json<EditCommandScript>
) -> Result<String> {
require_admin(is_admin, &username)?;
let mut tx = db.begin().await?;
sqlx::query!(
"UPDATE workspace_settings SET slack_command_script = $1 WHERE workspace_id = $2",
es.slack_command_script,
&w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"workspaces.edit_command_script",
ActionKind::Update,
&w_id,
Some(&authed.email.unwrap()),
Some([
("script", es.slack_command_script.unwrap_or("NO_SCRIPT".to_string()).as_str())
].into()),
)
.await?;
tx.commit().await?;
Ok(format!("Edit command script {}", &w_id))
}
async fn list_workspaces_as_super_admin(
authed: Authed,
Extension(user_db): Extension<UserDB>,
Query(pagination): Query<Pagination>,
Authed { email, .. }: Authed,
) -> JsonResult<Vec<Workspace>> {
let mut tx = user_db.begin(&authed).await?;
require_super_admin(&mut tx, email).await?;
let (per_page, offset) = crate::utils::paginate(pagination);
let workspaces = sqlx::query_as!(
Workspace,
"SELECT * FROM workspace LIMIT $1 OFFSET $2", per_page as i32, offset as i32)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(workspaces))
}
async fn user_workspaces(
Extension(db): Extension<DB>,
Authed { email, .. }: Authed,
) -> JsonResult<WorkspaceList> {
let email = email
.ok_or("not a personal token")
.map_err(|x| Error::NotAuthorized(x.to_string()))?;
let mut tx = db.begin().await?;
let workspaces = sqlx::query_as!(
UserWorkspace,
"SELECT workspace.id, workspace.name, usr.username
FROM workspace, usr WHERE usr.workspace_id = workspace.id AND usr.email = $1 AND deleted = false",
email)
.fetch_all(&mut tx)
.await?;
tx.commit().await?;
Ok(Json(WorkspaceList { email, workspaces }))
}
async fn create_workspace(
authed: Authed,
Extension(db): Extension<DB>,
Json(nw): Json<CreateWorkspace>
) -> Result<String> {
if &nw.username == "bot" {
return Err(Error::BadRequest("bot is a reserved username".to_string()))
}
let mut tx = db.begin().await?;
sqlx::query!(
"INSERT INTO workspace
(id, name, owner, domain)
VALUES ($1, $2, $3, $4)",
nw.id,
nw.name,
authed.email,
nw.domain
)
.execute(&mut tx)
.await?;
sqlx::query!(
"INSERT INTO workspace_settings
(workspace_id)
VALUES ($1)",
nw.id
)
.execute(&mut tx)
.await?;
let key = crate::utils::rd_string(64);
sqlx::query!(
"INSERT INTO workspace_key
(workspace_id, kind, key)
VALUES ($1, 'cloud', $2)",
nw.id, &key
)
.execute(&mut tx)
.await?;
let mc = magic_crypt::new_magic_crypt!(key, 256);
sqlx::query!(
"INSERT INTO variable
(workspace_id, path, value, is_secret, description)
VALUES ($1, 'g/all/pretty_secret', $2, true, 'This item is secret'),
($3, 'g/all/not_secret', $4, false, 'This item is not secret')",
nw.id,
crate::variables::encrypt(&mc, "pretty secret value".to_string()),
nw.id,
"finland does not actually exist",
)
.execute(&mut tx)
.await?;
sqlx::query!(
"INSERT INTO usr
(workspace_id, email, username, is_admin)
VALUES ($1, $2, $3, true)",
nw.id,
authed.email,
nw.username,
)
.execute(&mut tx)
.await?;
sqlx::query!(
"INSERT INTO group_
VALUES ($1, 'all', 'The group that always contains all users of this workspace')",
nw.id
)
.execute(&mut tx)
.await?;
sqlx::query!(
"INSERT INTO usr_to_group
VALUES ($1, 'all', $2)",
nw.id,
nw.username
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"workspaces.create",
ActionKind::Create,
&nw.id,
Some(&authed.email.unwrap()),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Created workspace {}", &nw.id))
}
async fn edit_workspace(
authed: Authed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Authed { is_admin, username, .. }: Authed,
Json(ew): Json<EditWorkspace>
) -> Result<String> {
require_admin(is_admin, &username)?;
let mut tx = db.begin().await?;
sqlx::query!(
"UPDATE workspace SET name = $1, owner = $2, domain = $3 WHERE id = $4",
ew.name,
ew.owner,
ew.domain,
&w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&authed.username,
"workspaces.update",
ActionKind::Update,
&w_id,
Some(&authed.email.unwrap()),
Some([
("domain", ew.domain.unwrap_or("NO_DOMAIN".to_string()).as_str())
].into()),
)
.await?;
tx.commit().await?;
Ok(format!("Updated workspace {}", &w_id))
}
async fn delete_workspace(
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Authed { is_admin, username, email, .. }: Authed,
) -> Result<String> {
require_admin(is_admin, &username)?;
let mut tx = db.begin().await?;
sqlx::query!(
"UPDATE workspace SET deleted = true WHERE id = $1",
&w_id
)
.execute(&mut tx)
.await?;
audit_log(
&mut tx,
&username,
"workspaces.delete",
ActionKind::Update,
&w_id,
Some(&email.unwrap_or("noemail".to_string())),
None,
)
.await?;
tx.commit().await?;
Ok(format!("Deleted workspace {}", &w_id))
}
async fn invite_user(
Authed { username, is_admin, .. }: Authed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(nu): Json<NewWorkspaceInvite>,
) -> Result<(StatusCode, String)> {
require_admin(is_admin, &username)?;
let mut tx = db.begin().await?;
sqlx::query!(
"INSERT INTO workspace_invite
(workspace_id, email, is_admin)
VALUES ($1, $2, $3)",
&w_id,
nu.email,
nu.is_admin
)
.execute(&mut tx)
.await?;
tx.commit().await?;
Ok((
StatusCode::CREATED,
format!("user with email {} invited", nu.email),
))
}
async fn delete_invite(
Authed { username, is_admin, .. }: Authed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
Json(nu): Json<NewWorkspaceInvite>,
) -> Result<(StatusCode, String)> {
require_admin(is_admin, &username)?;
let mut tx = db.begin().await?;
sqlx::query!(
"DELETE FROM workspace_invite WHERE
workspace_id = $1 AND email = $2 AND is_admin = $3",
&w_id,
nu.email,
nu.is_admin
)
.execute(&mut tx)
.await?;
tx.commit().await?;
Ok((
StatusCode::CREATED,
format!("invite to email {} deleted", nu.email),
))
}
async fn validate_username(
Extension(db): Extension<DB>,
Json(vu): Json<ValidateUsername>,
) -> Result<String> {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM usr WHERE username = $1 AND workspace_id = $2)",
vu.username,
vu.id
)
.fetch_one(&db)
.await?
.unwrap_or(true);
if exists {
return Err(Error::BadRequest("username already taken".to_string()))
}
Ok("valid username".to_string())
}
async fn validate_id(
Extension(db): Extension<DB>,
Json(wi): Json<WorkspaceId>,
) -> Result<String> {
let exists = sqlx::query_scalar!(
"SELECT EXISTS(SELECT 1 FROM workspace WHERE id = $1)",
wi.id
)
.fetch_one(&db)
.await?
.unwrap_or(true);
if exists {
return Err(Error::BadRequest("id already taken".to_string()))
}
Ok("valid workspace".to_string())
}
#[derive(Serialize)]
struct ScriptMetadata {
summary: String,
description: String,
schema: Option<Schema>,
is_template: bool,
lock: Vec<String>
}
async fn tarball_workspace(
authed: Authed,
Extension(db): Extension<DB>,
Path(w_id): Path<String>,
) -> Result<([(headers::HeaderName, String); 2], impl IntoResponse)> {
require_admin(authed.is_admin, &authed.username)?;
let tmp_dir = TempDir::new_in(".")?;
let name = format!("windmill-{w_id}.tar");
let file_path = tmp_dir.path().join(&name);
let file = File::create(&file_path).await?;
let mut a = tokio_tar::Builder::new(file);
{
let scripts = sqlx::query_as::<_, Script>(
"SELECT * FROM script as o WHERE workspace_id = $1 AND archived = false
AND created_at = (select max(created_at) from script where path = o.path AND workspace_id = $1)"
)
.bind(&w_id)
.fetch_all(&db)
.await?;
for script in scripts {
write_to_archive(script.content, format!("scripts/{}.py", script.path), &mut a).await?;
let lock = script.lock.unwrap_or_else(|| "".to_string())
.lines()
.map(|x| x.to_string())
.collect();
let metadata = ScriptMetadata {
summary: script.summary, description: script.description, schema: script.schema, is_template: script.is_template, lock };
let metadata_str = serde_json::to_string_pretty(&metadata).unwrap();
write_to_archive(metadata_str, format!("scripts/{}.json", script.path), &mut a).await?;
};
}
{
let resources = sqlx::query_as!(Resource,
"SELECT * FROM resource WHERE workspace_id = $1",
&w_id
)
.fetch_all(&db)
.await?;
for resource in resources {
let resource_str = serde_json::to_string_pretty(&resource).unwrap();
write_to_archive(resource_str, format!("resources/{}.json", resource.path), &mut a).await?;
}
}
{
let resource_types = sqlx::query_as!(ResourceType,
"SELECT * FROM resource_type WHERE workspace_id = $1",
&w_id
)
.fetch_all(&db)
.await?;
for resource_type in resource_types {
let resource_str = serde_json::to_string_pretty(&resource_type).unwrap();
write_to_archive(resource_str, format!("resource_types/{}.json", resource_type.name), &mut a).await?;
}
}
{
let flows = sqlx::query_as::<_, Flow>(
"SELECT * FROM flow WHERE workspace_id = $1 AND archived = false"
)
.bind(&w_id)
.fetch_all(&db)
.await?;
for flow in flows {
let flow_str = serde_json::to_string_pretty(&flow).unwrap();
write_to_archive(flow_str, format!("flows/{}.json", flow.path), &mut a).await?;
}
}
{
let variables = sqlx::query_as::<_, ListableVariable>(
"SELECT * FROM variable WHERE workspace_id = $1 AND is_secret = false"
)
.bind(&w_id)
.fetch_all(&db)
.await?;
for var in variables {
let flow_str = serde_json::to_string_pretty(&var).unwrap();
write_to_archive(flow_str, format!("variables/{}.json", var.path), &mut a).await?;
}
}
a.into_inner().await?;
let file = tokio::fs::File::open(file_path).await?;
let stream = ReaderStream::new(file);
let body = StreamBody::new(stream);
let headers = [
(header::CONTENT_TYPE, "application/x-tar".to_string()),
(
header::CONTENT_DISPOSITION,
format!("attachment; filename=\"{name}\""),
),
];
Ok((headers, body))
}
async fn write_to_archive(content: String, path: String, a: &mut tokio_tar::Builder<File>) -> Result<()> {
let bytes = content.as_bytes();
let mut header = tokio_tar::Header::new_gnu();
header.set_size(bytes.len() as u64);
header.set_mtime(0);
header.set_uid(0);
header.set_gid(0);
header.set_mode(0o777);
header.set_cksum();
a.append_data(&mut header, path, bytes).await?;
Ok(())
}

BIN
backend/v8.snap Normal file
View File

Binary file not shown.

28
community/README.md Normal file
View File

@@ -0,0 +1,28 @@
# Windmill community scripts
**Contributions welcome!**
The structure of the public templates are as follows:
- `starter/`:
- `resource_types/`: The resource types as
[jsonschema](https://json-schema.org/)
- `resources/u/bot/<foo>.json`: The resources
- `scripts/u/bot/<foo>.py`: The scripts code
- `scripts/u/bot/<foo>.json`: The corresponding script metadata
- `flows/u/bot/<foo>.json`: The flow entire definition
On merge to the main branch, the changes are automatically pushed to the
`starter` workspace of windmill.dev using the github action:
[windmill-gh-action-deploy](https://github.com/windmill-labs/windmill-gh-action-deploy).
Any element in the `starter` workspace is available read-only in every
workspace. Whenever an element is updated in `starter`, it is updated in every
workspaces, not just new ones.
This repo serves also as an example of how to setup a repo to back and
automatically deploy scripts and resources to your workspace using
[windmill-gh-action-deploy](https://github.com/windmill-labs/windmill-gh-action-deploy).
Indeed, it is sufficient to use the same folder layout and setup an action
similar to the one
[here](https://github.com/windmill-labs/windmill/blob/main/.github/workflows/deploy_to_windmill.yml).

32
community/resource_types/SMTP.json Executable file
View File

@@ -0,0 +1,32 @@
{
"workspace_id": "starter",
"name": "SMTP",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"host"
],
"properties": {
"host": {
"type": "string",
"default": "mail.smtpbucket.com",
"description": "SMTP host address"
},
"port": {
"type": "integer",
"default": 8025,
"description": "port number on which to connect"
},
"user": {
"type": "string",
"description": ""
},
"password": {
"type": "string",
"description": ""
}
}
},
"description": "SMTP connection info"
}

38
community/resource_types/mongodb.json Executable file
View File

@@ -0,0 +1,38 @@
{
"workspace_id": "starter",
"name": "mongodb",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"host",
"password",
"username"
],
"properties": {
"tls": {
"type": "boolean",
"description": ""
},
"host": {
"type": "string",
"default": "localhost:27017",
"description": "Hostname or IP address or Unix domain socket path of a single mongod or mongos instance to connect to, or a mongodb URI"
},
"port": {
"type": "integer",
"default": 27017,
"description": "port number on which to connect"
},
"password": {
"type": "string",
"description": ""
},
"username": {
"type": "string",
"description": ""
}
}
},
"description": "MongoDB connection info"
}

36
community/resource_types/mysql.json Executable file
View File

@@ -0,0 +1,36 @@
{
"workspace_id": "starter",
"name": "mysql",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"password",
"user"
],
"properties": {
"host": {
"type": "string",
"description": "The instance host"
},
"port": {
"type": "integer",
"default": 3306,
"description": "The instance port"
},
"user": {
"type": "string",
"description": "The postgres username"
},
"database": {
"type": "string",
"description": "The database to connect to"
},
"password": {
"type": "string",
"description": "The postgres users password"
}
}
},
"description": "MySQL connection info"
}

48
community/resource_types/postgres.json Executable file
View File

@@ -0,0 +1,48 @@
{
"workspace_id": "starter",
"name": "postgres",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"dbname",
"user",
"password"
],
"properties": {
"host": {
"type": "string",
"description": "The instance host"
},
"port": {
"type": "integer",
"description": "The instance port"
},
"user": {
"type": "string",
"description": "The postgres username"
},
"dbname": {
"type": "string",
"description": "The database name"
},
"sslmode": {
"enum": [
"disable",
"allow",
"prefer",
"require",
"verify-ca",
"verify-full"
],
"type": "string",
"description": "The sslmode"
},
"password": {
"type": "string",
"description": "The postgres users password"
}
}
},
"description": "A postgres database connection resource"
}

16
community/resource_types/slack.json Executable file
View File

@@ -0,0 +1,16 @@
{
"workspace_id": "starter",
"name": "slack",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [],
"properties": {
"token": {
"type": "string",
"description": "The slack token"
}
}
},
"description": "A slack token to interact with a specific workspace. Can be obtained from the OAuth integration in the workspace settings."
}

15
community/resources/g/all/demodb.json Executable file
View File

@@ -0,0 +1,15 @@
{
"workspace_id": "starter",
"path": "g/all/demodb",
"value": {
"host": "demodb.service.consul",
"port": "6543",
"user": "postgres",
"dbname": "demodb",
"sslmode": "disable",
"password": "demodb"
},
"description": "demodb",
"resource_type": "postgres",
"extra_perms": {}
}

33
community/scripts/u/bot/hello_world.json Executable file
View File

@@ -0,0 +1,33 @@
{
"summary": "Hello World",
"description": "The simplest hello world script",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [],
"properties": {
"name": {
"type": "string",
"default": "Nicolas Bourbaki",
"description": ""
}
}
},
"is_template": false,
"lock": [
"anyio==3.5.0",
"attrs==21.4.0",
"certifi==2021.10.8",
"charset-normalizer==2.0.12",
"h11==0.12.0",
"httpcore==0.14.7",
"httpx==0.21.3",
"idna==3.3",
"python-dateutil==2.8.2",
"rfc3986[idna2008]==1.5.0",
"six==1.16.0",
"sniffio==1.2.0",
"windmill-api==1.4.2",
"wmill==1.4.2"
]
}

6
community/scripts/u/bot/hello_world.py Executable file
View File

@@ -0,0 +1,6 @@
import wmill
def main(name: str = "Nicolas Bourbaki"):
print(f"Hello World and a warm welcome especially to {name}")
print("The env variable at `g/all/pretty_secret`:", wmill.get_variable("g/all/pretty_secret"))
return {"len": len(name), "splitted": name.split() }

26
community/scripts/u/bot/http_request.json Executable file
View File

@@ -0,0 +1,26 @@
{
"summary": "Do a simple HTTP post request",
"description": "This example pass a value as JSON data and use a secret as bearer token",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"my_value"
],
"properties": {
"my_value": {
"type": "string",
"default": null,
"description": ""
}
}
},
"is_template": true,
"lock": [
"certifi==2021.10.8",
"charset-normalizer==2.0.12",
"idna==3.3",
"requests==2.27.1",
"urllib3==1.26.9"
]
}

9
community/scripts/u/bot/http_request.py Executable file
View File

@@ -0,0 +1,9 @@
import requests
import os
def main(
my_value: str
):
headers={'Authorization': "Bearer: {}".format(os.environ.get("WM_TOKEN"))}
r = requests.post('https://httpbin.org/post', data={'key': my_value})
return r.text

42
community/scripts/u/bot/import_workspace_from_tarball.json Executable file
View File

@@ -0,0 +1,42 @@
{
"summary": "Import workspace from tarball",
"description": "Takes a tarball in and import scripts, resources, and resource types from a tarball exported of a workspace.\n\nIs used as webhook target for the windmill-gh-action-deploy",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"tarball"
],
"properties": {
"dry_run": {
"type": "boolean",
"default": false,
"description": ""
},
"tarball": {
"type": "string",
"default": null,
"description": "",
"contentEncoding": "base64"
}
}
},
"is_template": false,
"lock": [
"anyio==3.5.0",
"attrs==21.4.0",
"certifi==2021.10.8",
"charset-normalizer==2.0.12",
"h11==0.12.0",
"httpcore==0.14.7",
"httpx==0.22.0",
"idna==3.3",
"psycopg2-binary==2.9.3",
"python-dateutil==2.8.2",
"rfc3986[idna2008]==1.5.0",
"six==1.16.0",
"sniffio==1.2.0",
"windmill-api==1.2.0",
"wmill==1.2.0"
]
}

187
community/scripts/u/bot/import_workspace_from_tarball.py Executable file
View File

@@ -0,0 +1,187 @@
import tarfile
import io
import wmill
import json
from windmill_api.api.script import get_script_by_path, create_script
from windmill_api.models.create_script_json_body import CreateScriptJsonBody
from windmill_api.models.create_script_json_body_schema import (
CreateScriptJsonBodySchema,
)
from windmill_api.models.update_resource_json_body import UpdateResourceJsonBody
from windmill_api.models.create_resource_json_body import CreateResourceJsonBody
from windmill_api.models.update_resource_type_json_body import (
UpdateResourceTypeJsonBody,
)
from windmill_api.models.create_resource_type_json_body import (
CreateResourceTypeJsonBody,
)
from windmill_api.api.resource import (
create_resource,
create_resource_type,
update_resource,
update_resource_type,
get_resource,
get_resource_type,
)
client = wmill.Client()
SCRIPTS_PREFIX = "scripts/"
RESOURCES_PREFIX = "resources/"
RESOURCE_TYPES_PREFIX = "resource_types/"
def main(tarball: bytes, dry_run: bool = False):
print("Tarball of size: {} bytes".format(len(tarball)))
io_bytes = io.BytesIO(tarball)
tar = tarfile.open(fileobj=io_bytes, mode="r")
names = tar.getnames()
for m in tar.getmembers():
if m.name.startswith(SCRIPTS_PREFIX) and m.name.endswith(".py"):
path = m.name[len(SCRIPTS_PREFIX) : -len(".py")]
print("Processing script {}, {}".format(path, m.name))
get_script_response = get_script_by_path.sync_detailed(
workspace=client.workspace, path=path, client=client.client
)
json_tar_path = "{}.json".format(m.name[: -len(".py")])
has_json = json_tar_path in names
parent_hash = None
summary = None
description = None
is_template = None
schema = None
content = tar.extractfile(m).read().decode("utf-8")
if get_script_response.status_code == 200:
old_script = json.loads(get_script_response.content)
if old_script["content"] == content:
if not has_json:
print(
"same content and no metadata for this script in tarball, no need to update"
)
continue
json_content = tar.extractfile(tar.getmember(json_tar_path)).read()
metadata = json.loads(json_content)
if (
old_script.get("summary") == metadata.get("summary")
and old_script.get("description") == metadata.get("description")
and old_script.get("is_template") == metadata.get("is_template")
and old_script.get("schema") == metadata.get("schema")
and old_script.get("lock") == metadata.get("lock")
):
print("same content and no metadata, no need to update")
continue
parent_hash = old_script["hash"]
summary = old_script.get("summary")
description = old_script.get("description")
is_template = old_script.get("is_template")
schema = old_script.get("schema")
lock = None
if has_json:
json_content = tar.extractfile(tar.getmember(json_tar_path)).read()
metadata = json.loads(json_content)
summary = metadata.get("summary")
description = metadata.get("description")
is_template = metadata.get("is_template")
schema = metadata.get("schema")
lock = metadata.get("lock")
if lock == []:
lock = None
if schema:
schema = CreateScriptJsonBodySchema.from_dict(schema)
print("Uploading new version of script at path: {}".format(path))
if dry_run:
print("Skipped because dry-run")
else:
r = create_script.sync_detailed(
client.workspace,
client=client.client,
json_body=CreateScriptJsonBody(
content=content,
path=path,
parent_hash=parent_hash,
summary=summary,
description=description,
is_template=is_template,
schema=schema,
lock=lock,
),
)
print(r.content)
if m.name.startswith(RESOURCES_PREFIX) and m.name.endswith(".json"):
path = m.name[len(RESOURCES_PREFIX) : -len(".json")]
print("Processing resource {}, {}".format(path, m.name))
get_resource_response = get_resource.sync_detailed(
workspace=client.workspace, path=path, client=client.client
)
content = tar.extractfile(m).read().decode("utf-8")
resource = json.loads(content)
if get_resource_response.status_code == 200:
old_resource = json.loads(get_resource_response.content)
if resource["value"] != old_resource["value"]:
print("Updating existing resource")
r = update_resource.sync_detailed(
client.workspace,
path=path,
json_body=UpdateResourceJsonBody.from_dict(resource),
client=client.client,
)
print(r)
else:
print("Skipping updating identical resource")
else:
print("Creating new resource")
r = create_resource.sync_detailed(
client.workspace,
json_body=CreateResourceJsonBody.from_dict(resource),
client=client.client,
)
print(r)
if m.name.startswith(RESOURCE_TYPES_PREFIX) and m.name.endswith(".json"):
path = m.name[len(RESOURCE_TYPES_PREFIX) : -len(".json")]
print("Processing resource type {}, {}".format(path, m.name))
get_resource_response = get_resource_type.sync_detailed(
workspace=client.workspace, path=path, client=client.client
)
content = tar.extractfile(m).read().decode("utf-8")
resource = json.loads(content)
print(resource)
if get_resource_response.status_code == 200:
old_resource_type = json.loads(get_resource_response.content)
if resource["schema"] != old_resource_type["schema"]:
print("Updating existing resource type")
r = update_resource_type.sync_detailed(
client.workspace,
path=path,
json_body=UpdateResourceTypeJsonBody.from_dict(resource),
client=client.client,
)
print(r)
else:
print("Skipping updating identical resource type")
else:
print("Creating new resource type")
r = create_resource_type.sync_detailed(
client.workspace,
json_body=CreateResourceTypeJsonBody.from_dict(resource),
client=client.client,
)
print(r)

56
community/scripts/u/bot/message_slack.json Executable file
View File

@@ -0,0 +1,56 @@
{
"summary": "Send a slack message",
"description": "Send a message to a channel or person in the connected slack workspace",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"slack_resource",
"text"
],
"properties": {
"slack_resource": {
"description": "",
"type": "object",
"default": null,
"format": "resource-slack"
},
"text": {
"type": "string",
"default": null,
"description": "",
"format": ""
},
"channel": {
"type": "string",
"default": null,
"description": "",
"format": ""
},
"user": {
"type": "string",
"default": null,
"description": "",
"format": ""
}
}
},
"is_template": false,
"lock": [
"anyio==3.5.0",
"attrs==21.4.0",
"certifi==2021.10.8",
"charset-normalizer==2.0.12",
"h11==0.12.0",
"httpcore==0.14.7",
"httpx==0.21.3",
"idna==3.3",
"python-dateutil==2.8.2",
"rfc3986[idna2008]==1.5.0",
"six==1.16.0",
"slack-sdk==3.15.2",
"sniffio==1.2.0",
"windmill-api==1.5.0",
"wmill==1.5.0"
]
}

24
community/scripts/u/bot/message_slack.py Executable file
View File

@@ -0,0 +1,24 @@
import os
import wmill
from slack_sdk.web.client import WebClient
def main(
slack_resource: dict,
text: str,
channel: str = None,
user: str = None,
):
slack_client = WebClient(token=slack_resource["token"])
if channel == "":
channel = None
if user == "":
user = None
if channel is None and user is None or (channel is not None and user is not None):
raise Exception("one and only one of channel or user need to be set")
if user is not None:
channel = "@{}".format(user)
slack_client.chat_postMessage(channel=channel, text=text)

43
community/scripts/u/bot/query_postgres.json Executable file
View File

@@ -0,0 +1,43 @@
{
"summary": "Execute an SQL query on a postgres resource",
"description": "An example of how to use resources from scripts. ",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"pg_con"
],
"properties": {
"pg_con": {
"description": "",
"type": "object",
"default": null,
"format": "resource-postgres"
},
"sql_query": {
"description": "",
"type": "string",
"default": "SELECT * from demo"
}
}
},
"is_template": false,
"lock": [
"anyio==3.5.0",
"attrs==21.4.0",
"certifi==2021.10.8",
"charset-normalizer==2.0.12",
"h11==0.12.0",
"httpcore==0.14.7",
"httpx==0.21.3",
"idna==3.3",
"psycopg2-binary==2.9.3",
"python-dateutil==2.8.2",
"rfc3986[idna2008]==1.5.0",
"six==1.16.0",
"sniffio==1.2.0",
"windmill-api==1.5.0",
"wmill==1.5.0",
"wmill-pg==1.5.0"
]
}

12
community/scripts/u/bot/query_postgres.py Executable file
View File

@@ -0,0 +1,12 @@
import os
from wmill_pg import query
def main(
pg_con: dict, # a resource of type postgres (constrained at step 3: UI customisation)
sql_query: str = "SELECT * from demo"
):
# query that returns rows will return them as a list
return query(sql_query, pg_con)

33
community/scripts/u/bot/return_chart_as_image.json Executable file
View File

@@ -0,0 +1,33 @@
{
"summary": "Build seaborn charts and return a rendered image",
"description": "This script showcase that image can be rendered from their base64 encoding when their dict key is \"png\"",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [],
"properties": {
"seed": {
"type": "integer",
"default": 1234,
"description": ""
}
}
},
"is_template": false,
"lock": [
"cycler==0.11.0",
"fonttools==4.33.3",
"kiwisolver==1.4.2",
"matplotlib==3.5.2",
"numpy==1.22.3",
"packaging==21.3",
"pandas==1.4.2",
"pillow==9.1.0",
"pyparsing==3.0.8",
"python-dateutil==2.8.2",
"pytz==2022.1",
"scipy==1.8.0",
"seaborn==0.11.2",
"six==1.16.0"
]
}

22
community/scripts/u/bot/return_chart_as_image.py Executable file
View File

@@ -0,0 +1,22 @@
import seaborn as sns
import base64
import numpy as np
import pandas as pd
def main(
seed: int = 1234
):
np.random.seed(seed)
data = np.random.multivariate_normal([0, 0], [[5, 2], [2, 2]], size=2000)
data = pd.DataFrame(data, columns=['x', 'y'])
file_output = "output.png"
with sns.axes_style('white'):
plot = sns.jointplot("x", "y", data, kind='hex')
fig = plot.figure.savefig(file_output)
with open(file_output, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode('ascii')
return {"png": encoded_string}

23
community/scripts/u/bot/return_table.json Executable file
View File

@@ -0,0 +1,23 @@
{
"summary": "Return Table",
"description": "An example of a script that return a table that can get rendered",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [],
"properties": {
"seed": {
"default": 123,
"description": ""
}
}
},
"is_template": false,
"lock": [
"numpy==1.22.3",
"pandas==1.4.2",
"python-dateutil==2.8.2",
"pytz==2022.1",
"six==1.16.0"
]
}

11
community/scripts/u/bot/return_table.py Executable file
View File

@@ -0,0 +1,11 @@
import numpy as np
import pandas as pd
def main(
seed = 123
):
np.random.seed(seed)
df = pd.DataFrame(np.random.randint(0,100,size=(100, 4)), columns=list('ABCD'))
print(df)
print("See the result tab to see it rendered as a table")
return [df.columns.values.tolist()] + df.values.tolist()

57
community/scripts/u/bot/send_email_gmail.json Executable file
View File

@@ -0,0 +1,57 @@
{
"summary": "Send an email (smtp and gmail)",
"description": "Send an email using smtplib and in particular with a gmail account set with a [secure app password](https://support.google.com/accounts/answer/185833?hl=en).\n\nFor gmail the following smtp configuration set in resource would work:\n```json\n{\n \"host\": \"smtp.gmail.com\",\n \"password\": \"$var:u/user/your_app_password\",\n \"port\": 587,\n \"user\": \"youremail@gmail.com\"\n}\n```\nwhere you have set the variable `u/you/your_secure_password` as a variable containing the app password.",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"smtp_config",
"to",
"subject",
"body"
],
"properties": {
"smtp_config": {
"description": "",
"type": "object",
"default": null,
"format": "resource-SMTP"
},
"to": {
"type": "string",
"default": null,
"description": "",
"format": ""
},
"subject": {
"type": "string",
"default": null,
"description": "",
"format": ""
},
"body": {
"type": "string",
"default": null,
"description": "",
"format": ""
}
}
},
"is_template": true,
"lock": [
"anyio==3.5.0",
"attrs==21.4.0",
"certifi==2021.10.8",
"charset-normalizer==2.0.12",
"h11==0.12.0",
"httpcore==0.14.7",
"httpx==0.21.3",
"idna==3.3",
"python-dateutil==2.8.2",
"rfc3986[idna2008]==1.5.0",
"six==1.16.0",
"sniffio==1.2.0",
"windmill-api==1.5.0",
"wmill==1.5.0"
]
}

19
community/scripts/u/bot/send_email_gmail.py Executable file
View File

@@ -0,0 +1,19 @@
from wmill import get_resource
import smtplib
def main(smtp_config: dict, to: str, subject: str, body: str):
server = smtplib.SMTP(host=smtp_config["host"], port=smtp_config["port"])
server.ehlo()
server.starttls()
server.login(smtp_config["user"], smtp_config["password"])
server.sendmail(
smtp_config["user"],
to,
f"""Subject: {subject}
{body}
""")

43
community/scripts/u/bot/send_slack_image.json Executable file
View File

@@ -0,0 +1,43 @@
{
"summary": "Send image to slack",
"description": "Send a base64 image to a slack channel or user. Choose one of user or channel but not both.",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"slack_resource",
"img_data"
],
"properties": {
"slack_resource": {
"description": "",
"type": "object",
"default": null,
"format": "resource-slack"
},
"img_data": {
"type": "string",
"default": null,
"description": "",
"contentEncoding": "base64",
"format": ""
},
"channel": {
"type": "string",
"default": null,
"description": "",
"format": ""
},
"user": {
"type": "string",
"default": null,
"description": "",
"format": ""
}
}
},
"is_template": false,
"lock": [
"slack-sdk==3.15.2"
]
}

37
community/scripts/u/bot/send_slack_image.py Executable file
View File

@@ -0,0 +1,37 @@
import os
from slack_sdk.web.client import WebClient
from datetime import date, datetime
def main(
slack_resource: dict,
img_data: bytes,
channel: str = None,
user: str = None,
):
slack_client = WebClient(token=slack_resource["token"])
if channel == "":
channel = None
if user == "":
user = None
if channel is None and user is None or (channel is not None and user is not None):
raise Exception("one and only one of channel or user need to be set")
if user is not None:
channel = "@{}".format(user)
tmp_image = "image.png"
with open(tmp_image, "wb") as fh:
fh.write(img_data)
slack_client.files_upload(
file=tmp_image, initial_comment="Weekly report", channels=channel
)
now = datetime.now()
current_time = now.strftime("%H:%M")
today = date.today()
print("Sent to slack successfully on", today, current_time)

49
community/scripts/u/bot/table_to_pie.json Executable file
View File

@@ -0,0 +1,49 @@
{
"summary": "Table to pie plot",
"description": "Transform a table of rows into a pieplot image and return its base64",
"schema": {
"type": "object",
"$schema": "https://json-schema.org/draft/2020-12/schema",
"required": [
"rows"
],
"properties": {
"rows": {
"type": "array",
"default": null,
"description": ""
},
"title": {
"type": "string",
"default": "",
"description": ""
}
}
},
"is_template": false,
"lock": [
"anyio==3.5.0",
"attrs==21.4.0",
"certifi==2021.10.8",
"charset-normalizer==2.0.12",
"cycler==0.11.0",
"fonttools==4.32.0",
"h11==0.12.0",
"httpcore==0.14.7",
"httpx==0.22.0",
"idna==3.3",
"kiwisolver==1.4.2",
"matplotlib==3.5.1",
"numpy==1.22.3",
"packaging==21.3",
"pillow==9.1.0",
"psycopg2-binary==2.9.3",
"pyparsing==3.0.8",
"python-dateutil==2.8.2",
"rfc3986[idna2008]==1.5.0",
"six==1.16.0",
"sniffio==1.2.0",
"windmill-api==1.2.0",
"wmill==1.2.0"
]
}

38
community/scripts/u/bot/table_to_pie.py Executable file
View File

@@ -0,0 +1,38 @@
import wmill
import matplotlib.pyplot as plt
import base64
from datetime import date
client = wmill.Client()
def main(rows: list, title: str = ""):
file_output = "output.png"
fig, ax = plt.subplots(figsize=(6, 3), subplot_kw=dict(aspect="equal"))
# Get labels and data from list
labels = [x[0] for x in rows]
data = [x[1] for x in rows]
# Create donut chart
plt.pie(
data,
labels=labels,
autopct="%.0f%%",
wedgeprops={"linewidth": 5, "edgecolor": "white"},
)
my_circle = plt.Circle((0, 0), 0.38, color="white")
p = plt.gcf()
p.gca().add_artist(my_circle)
ax.set_title(title)
# Create png
plt.savefig(file_output)
with open(file_output, "rb") as image_file:
encoded_string = base64.b64encode(image_file.read()).decode("ascii")
return {"png": encoded_string}

Some files were not shown because too many files have changed in this diff Show More