feat: generate script summary (#3110)

* feat: generate script summary

* fix: design nits

* feat: better ui + descriptions

frontend/src/lib/components/ScriptBuilder.svelte

@@ -51,6 +51,8 @@
 	import { cloneDeep } from 'lodash'
 	import type Editor from './Editor.svelte'
 	import WorkerTagPicker from './WorkerTagPicker.svelte'
+	import MetadataGen from './copilot/MetadataGen.svelte'
+	import MetadataGenToggle from './copilot/MetadataGenToggle.svelte'
 
 	export let script: NewScript
 	export let initialPath: string = ''
@@ -367,6 +369,8 @@
 	let dirtyPath = false
 
 	let selectedTab: 'metadata' | 'runtime' | 'ui' = 'metadata'
+
+	let descriptionTextArea: HTMLTextAreaElement | undefined
 </script>
 
 <svelte:window on:keydown={onKeyDown} />
@@ -393,6 +397,9 @@
 						<TabContent value="metadata">
 							<div class="flex flex-col gap-8">
 								<Section label="Metadata">
+									<svelte:fragment slot="header">
+										<MetadataGenToggle />
+									</svelte:fragment>
 									<svelte:fragment slot="action">
 										<div class="flex flex-row items-center gap-2">
 											<ErrorHandlerToggleButton
@@ -404,25 +411,34 @@
 										</div>
 									</svelte:fragment>
 									<div class="flex flex-col gap-4">
-										<Label label="Summary">
+										<MetadataGen
+											label="Summary"
+											bind:content={script.summary}
+											lang={script.language}
+											code={script.content}
+											configName="summary"
+											let:updateFocus
+											on:change={() => {
+												if (initialPath == '' && script.summary?.length > 0 && !dirtyPath) {
+													path?.setName(
+														script.summary
+															.toLowerCase()
+															.replace(/[^a-z0-9_]/g, '_')
+															.replace(/-+/g, '_')
+															.replace(/^-|-$/g, '')
+													)
+												}
+											}}
+										>
 											<input
 												type="text"
 												autofocus
+												on:focus={() => updateFocus(true)}
+												on:blur={() => updateFocus(false)}
 												bind:value={script.summary}
-												placeholder="Short summary to be displayed when listed"
-												on:keyup={() => {
-													if (initialPath == '' && script.summary?.length > 0 && !dirtyPath) {
-														path?.setName(
-															script.summary
-																.toLowerCase()
-																.replace(/[^a-z0-9_]/g, '_')
-																.replace(/-+/g, '_')
-																.replace(/^-|-$/g, '')
-														)
-													}
-												}}
+												placeholder={'Short summary to be displayed when listed'}
 											/>
-										</Label>
+										</MetadataGen>
 										<Label label="Path">
 											<Path
 												bind:this={path}
@@ -435,14 +451,25 @@
 												kind="script"
 											/>
 										</Label>
-										<Label label="Description">
+										<MetadataGen
+											bind:content={script.description}
+											lang={script.language}
+											code={script.content}
+											configName="description"
+											el={descriptionTextArea}
+											label="Description"
+											let:updateFocus
+										>
 											<textarea
+												on:focus={() => updateFocus(true)}
+												on:blur={() => updateFocus(false)}
 												use:autosize
+												bind:this={descriptionTextArea}
 												bind:value={script.description}
-												placeholder="Description displayed in the details page"
+												placeholder={'Description displayed in the details page'}
 												class="text-sm"
 											/>
-										</Label>
+										</MetadataGen>
 									</div>
 								</Section>
 

frontend/src/lib/components/copilot/MetadataGen.svelte

@@ -0,0 +1,160 @@
+<script lang="ts">
+	import { getCompletion } from './lib'
+	import type { NewScript } from '$lib/gen'
+	import { isInitialCode } from '$lib/script_helpers'
+	import { Loader2 } from 'lucide-svelte'
+	import type { ChatCompletionMessageParam } from 'openai/resources'
+	import { copilotInfo, metadataCompletionEnabled } from '$lib/stores'
+	import Label from '../Label.svelte'
+	import { createEventDispatcher } from 'svelte'
+
+	type Config = {
+		system: string
+		user: string
+		mode: 'automatic' | 'manual'
+	}
+
+	const configs: {
+		summary: Config
+		description: Config
+	} = {
+		summary: {
+			system: `
+You are a helpful AI assistant. You generate very brief summaries from scripts.
+The summaries need to be as short as possible (maximum 8 words) and only give a global idea. Do not specify the programming language. Do not use any punctation. Avoid using prepositions and articles.
+Examples: List the commits of a GitHub repository, Divide a number by 16, etc..
+`,
+			user: `
+		Generate a very short summary for the script below:
+\'\'\'{lang}
+{code}
+\`\`\`
+`,
+			mode: 'automatic'
+		},
+		description: {
+			system: `
+You are a helpful AI assistant. You generate descriptions from scripts.
+These descriptions are used to explain to other users what the script does, on particular on the input and what it returns.
+Descriptions should contain a maximum of 4-5 sentences.
+The description should focus on what it does and should not contain what concepts it uses (e.g. function named main, export an async function, etc...)
+`,
+			user: `
+		Generate a description for the script below:
+\'\'\'{lang}
+{code}
+\`\`\`
+`,
+			mode: 'manual'
+		}
+	}
+
+	export let content: string | undefined
+	export let code: string
+	export let lang: NewScript.language
+	export let configName: keyof typeof configs
+
+	export let el: HTMLElement | undefined = undefined
+	export let label: string
+
+	let loading = false
+	let abortController = new AbortController()
+
+	let focused = false
+	const updateFocus = (val) => {
+		focused = val
+	}
+
+	let config: Config = configs[configName]
+
+	async function generateContent() {
+		abortController = new AbortController()
+		loading = true
+		try {
+			const messages: ChatCompletionMessageParam[] = [
+				{
+					role: 'system',
+					content: config.system
+				},
+				{
+					role: 'user',
+					content: config.user.replace('{lang}', lang).replace('{code}', code)
+				}
+			]
+			const response = await getCompletion(messages, abortController)
+			content = ''
+			for await (const chunk of response) {
+				const toks = chunk.choices[0]?.delta?.content || ''
+				content += toks
+				if (el !== undefined) {
+					el.style.height = 'auto'
+					el.style.height = el.scrollHeight + 'px'
+				}
+			}
+		} catch (err) {
+			console.error('Could not generate summary', err)
+		} finally {
+			loading = false
+		}
+	}
+
+	if (
+		$copilotInfo.exists_openai_resource_path &&
+		$metadataCompletionEnabled &&
+		config.mode === 'automatic' &&
+		code &&
+		!content &&
+		!isInitialCode(code)
+	) {
+		generateContent()
+	}
+
+	const dispatch = createEventDispatcher()
+
+	$: if (content) {
+		dispatch('change', { content })
+	}
+</script>
+
+<div class="relative">
+	<div class="flex flex-row" />
+	<!-- svelte-ignore a11y-no-static-element-interactions -->
+	<Label {label}>
+		<div slot="header" class="flex flex-row pl-1 gap-2 items-center">
+			{#if $copilotInfo.exists_openai_resource_path && $metadataCompletionEnabled}
+				{#if loading}
+					<Loader2 class="animate-spin text-gray-400" size={18} />
+					<span class="text-xs">
+						<span class="border px-1 py-0.5 rounded-md text-2xs text-bold bg-white text-black">
+							ESC
+						</span> to cancel
+					</span>
+				{:else if !content && focused}
+					<span class="text-xs"
+						>0
+						<span class="border px-1 py-0.5 rounded-md text-2xs text-bold bg-white text-black">
+							TAB
+						</span> to generate
+					</span>
+				{/if}
+			{/if}
+		</div>
+		<div
+			on:keydown={(event) => {
+				if (!$copilotInfo.exists_openai_resource_path || !$metadataCompletionEnabled) {
+					return
+				}
+				if (event.key === 'Tab' && !loading && !content) {
+					event.preventDefault()
+					generateContent()
+				} else if (event.key === 'Escape' && loading) {
+					event.preventDefault()
+					event.stopPropagation()
+					abortController.abort()
+				}
+			}}
+		>
+			<slot {updateFocus} />
+		</div>
+	</Label>
+</div>

frontend/src/lib/components/copilot/MetadataGenToggle.svelte

@@ -0,0 +1,55 @@
+<script lang="ts">
+	import { ExternalLink, ZapIcon, ZapOffIcon } from 'lucide-svelte'
+	import Button from '../common/button/Button.svelte'
+	import { copilotInfo, metadataCompletionEnabled } from '$lib/stores'
+	import Popover from '../Popover.svelte'
+	import { getLocalSetting, storeLocalSetting } from '$lib/utils'
+
+	const SETTING_NAME = 'metadataCompletionEnabled'
+	function loadSetting() {
+		$metadataCompletionEnabled = (getLocalSetting(SETTING_NAME) ?? 'true') == 'true'
+	}
+
+	function storeSetting() {
+		$metadataCompletionEnabled = !$metadataCompletionEnabled
+		storeLocalSetting(SETTING_NAME, $metadataCompletionEnabled.toString())
+	}
+
+	loadSetting()
+</script>
+
+{#if $copilotInfo.exists_openai_resource_path}
+	<Popover>
+		<svelte:fragment slot="text"
+			>{$metadataCompletionEnabled ? 'Disable' : 'Enable'} metadata completion (applies only to you)</svelte:fragment
+		>
+		<Button
+			color="light"
+			startIcon={{
+				icon: $metadataCompletionEnabled ? ZapIcon : ZapOffIcon
+			}}
+			on:click={() => {
+				storeSetting()
+			}}
+		/>
+	</Popover>
+{:else}
+	<Popover>
+		<svelte:fragment slot="text">
+			Enable Windmill AI in the <a
+				href="/workspace_settings?tab=openai"
+				target="_blank"
+				class="inline-flex flex-row items-center gap-1"
+			>
+				workspace settings <ExternalLink size={16} />
+			</a>
+		</svelte:fragment>
+		<Button
+			color="light"
+			startIcon={{
+				icon: ZapOffIcon
+			}}
+			disabled
+		/>
+	</Popover>
+{/if}

frontend/src/lib/components/copilot/lib.ts

@@ -233,7 +233,7 @@ const PROMPTS_CONFIGS = {
 export async function getNonStreamingCompletion(
 	messages: ChatCompletionMessageParam[],
 	abortController: AbortController,
-	model: string = 'gpt-4-1106-preview',
+	model = openaiConfig.model,
 	noCache?: boolean
 ) {
 	const openaiClient = workspacedOpenai.getClient()
@@ -262,13 +262,15 @@ export async function getNonStreamingCompletion(
 
 export async function getCompletion(
 	messages: ChatCompletionMessageParam[],
-	abortController: AbortController
+	abortController: AbortController,
+	model = openaiConfig.model
 ) {
 	const openaiClient = workspacedOpenai.getClient()
 	const completion = await openaiClient.chat.completions.create(
 		{
 			...openaiConfig,
-			messages
+			messages,
+			model
 		},
 		{
 			signal: abortController.signal

frontend/src/lib/stores.ts

@@ -75,6 +75,7 @@ export const copilotInfo = writable<{
 })
 export const codeCompletionLoading = writable<boolean>(false)
 export const codeCompletionSessionEnabled = writable<boolean>(true)
+export const metadataCompletionEnabled = writable<boolean>(true)
 export const formatOnSave = writable<boolean>(true)
 
 type SQLBaseSchema = {

llm/src/gen_samples.py

@@ -121,7 +121,7 @@ def gen_samples(queries_path: str, answers_path: str, prompts_path: str):
         )
         client = openai.OpenAI()
         chat_completion = client.chat.completions.create(
-            model="gpt-4-1106-preview",
+            model="gpt-4-0125-preview",
             messages=[
                 {"role": "system", "content": system},
                 {"role": "user", "content": prompt},