Merge remote-tracking branch 'origin/new-page-framework' into llm_framework

This commit is contained in:
aries_ckt 2023-06-30 16:36:27 +08:00
commit 0a382b64db
3 changed files with 553 additions and 335 deletions

View File

@ -1,13 +1,21 @@
'use client' 'use client'
import { useSearchParams } from 'next/navigation' import { useSearchParams, useRouter } from 'next/navigation'
import React, { useState, useEffect } from 'react' import React, { useState, useEffect } from 'react'
import { useColorScheme, Table, Stack } from '@/lib/mui' import {
useColorScheme,
Table,
Stack,
Typography,
Breadcrumbs,
Link
} from '@/lib/mui'
import { Popover, Pagination } from 'antd' import { Popover, Pagination } from 'antd'
import { fetchBaseURL } from '@/app/datastores/constants' import { fetchBaseURL } from '@/app/datastores/constants'
const page_size = 20 const page_size = 20
const ChunkList = () => { const ChunkList = () => {
const router = useRouter()
const { mode } = useColorScheme() const { mode } = useColorScheme()
const spaceName = useSearchParams().get('spacename') const spaceName = useSearchParams().get('spacename')
const documentId = useSearchParams().get('documentid') const documentId = useSearchParams().get('documentid')
@ -41,99 +49,137 @@ const ChunkList = () => {
}, []) }, [])
return ( return (
<div className="p-4"> <div className="p-4">
{chunkList.length ? ( <Stack
<> direction="row"
<Table justifyContent="flex-start"
color="primary" alignItems="center"
variant="plain" sx={{ marginBottom: '20px' }}
size="lg" >
sx={{ <Breadcrumbs aria-label="breadcrumbs">
'& tbody tr: hover': { <Link
backgroundColor: onClick={() => {
mode === 'light' ? 'rgb(246, 246, 246)' : 'rgb(33, 33, 40)' router.push('/datastores')
},
'& tbody tr: hover a': {
textDecoration: 'underline'
}
}} }}
key="Knowledge Space"
underline="hover"
color="neutral"
fontSize="inherit"
> >
<thead> Knowledge Space
<tr> </Link>
<th>Name</th> <Link
<th>Content</th> onClick={() => {
<th>Meta Data</th> router.push(`/datastores/documents?name=${spaceName}`)
</tr>
</thead>
<tbody>
{chunkList.map((row: any) => (
<tr key={row.id}>
<td>{row.doc_name}</td>
<td>
{
<Popover content={row.content} trigger="hover">
{row.content.length > 10
? `${row.content.slice(0, 10)}...`
: row.content}
</Popover>
}
</td>
<td>
{
<Popover
content={JSON.stringify(row.meta_info || '{}', null, 2)}
trigger="hover"
>
{row.meta_info.length > 10
? `${row.meta_info.slice(0, 10)}...`
: row.meta_info}
</Popover>
}
</td>
</tr>
))}
</tbody>
</Table>
<Stack
direction="row"
justifyContent="flex-end"
sx={{
marginTop: '20px'
}} }}
key="Knowledge Space"
underline="hover"
color="neutral"
fontSize="inherit"
> >
<Pagination Documents
defaultPageSize={20} </Link>
showSizeChanger={false} <Typography fontSize="inherit">Chunks</Typography>
current={current} </Breadcrumbs>
total={total} </Stack>
onChange={async (page) => { <div className="p-4">
const res = await fetch( {chunkList.length ? (
`${fetchBaseURL}/knowledge/${spaceName}/chunk/list`, <>
{ <Table
method: 'POST', color="primary"
headers: { variant="plain"
'Content-Type': 'application/json' size="lg"
}, sx={{
body: JSON.stringify({ '& tbody tr: hover': {
document_id: documentId, backgroundColor:
page, mode === 'light' ? 'rgb(246, 246, 246)' : 'rgb(33, 33, 40)'
page_size },
}) '& tbody tr: hover a': {
} textDecoration: 'underline'
)
const data = await res.json()
if (data.success) {
setChunkList(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
} }
}} }}
hideOnSinglePage >
/> <thead>
</Stack> <tr>
</> <th>Name</th>
) : ( <th>Content</th>
<></> <th>Meta Data</th>
)} </tr>
</thead>
<tbody>
{chunkList.map((row: any) => (
<tr key={row.id}>
<td>{row.doc_name}</td>
<td>
{
<Popover content={row.content} trigger="hover">
{row.content.length > 10
? `${row.content.slice(0, 10)}...`
: row.content}
</Popover>
}
</td>
<td>
{
<Popover
content={JSON.stringify(
row.meta_info || '{}',
null,
2
)}
trigger="hover"
>
{row.meta_info.length > 10
? `${row.meta_info.slice(0, 10)}...`
: row.meta_info}
</Popover>
}
</td>
</tr>
))}
</tbody>
</Table>
<Stack
direction="row"
justifyContent="flex-end"
sx={{
marginTop: '20px'
}}
>
<Pagination
defaultPageSize={20}
showSizeChanger={false}
current={current}
total={total}
onChange={async (page) => {
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/chunk/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
document_id: documentId,
page,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setChunkList(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
}}
hideOnSinglePage
/>
</Stack>
</>
) : (
<></>
)}
</div>
</div> </div>
) )
} }

View File

@ -13,10 +13,15 @@ import {
Input, Input,
Textarea, Textarea,
Chip, Chip,
Switch,
Typography,
Breadcrumbs,
Link,
styled styled
} from '@/lib/mui' } from '@/lib/mui'
import moment from 'moment' import moment from 'moment'
import { InboxOutlined } from '@ant-design/icons' import { InboxOutlined } from '@ant-design/icons'
import CheckCircleOutlinedIcon from '@mui/icons-material/CheckCircleOutlined';
import type { UploadProps } from 'antd' import type { UploadProps } from 'antd'
import { Upload, Pagination, Popover, message } from 'antd' import { Upload, Pagination, Popover, message } from 'antd'
import { fetchBaseURL } from '@/app/datastores/constants' import { fetchBaseURL } from '@/app/datastores/constants'
@ -33,8 +38,8 @@ const Item = styled(Sheet)(({ theme }) => ({
color: theme.vars.palette.text.secondary color: theme.vars.palette.text.secondary
})) }))
const stepsOfAddingDocument = [ const stepsOfAddingDocument = [
'1.Choose a Datasource type', 'Choose a Datasource type',
'2.Setup the Datasource' 'Setup the Datasource'
] ]
const documentTypeList = [ const documentTypeList = [
{ {
@ -72,6 +77,7 @@ const Documents = () => {
const [originFileObj, setOriginFileObj] = useState<any>(null) const [originFileObj, setOriginFileObj] = useState<any>(null)
const [total, setTotal] = useState<number>(0) const [total, setTotal] = useState<number>(0)
const [current, setCurrent] = useState<number>(0) const [current, setCurrent] = useState<number>(0)
const [synchChecked, setSynchChecked] = useState<boolean>(true)
const props: UploadProps = { const props: UploadProps = {
name: 'file', name: 'file',
multiple: false, multiple: false,
@ -112,20 +118,33 @@ const Documents = () => {
}, []) }, [])
return ( return (
<div className="p-4"> <div className="p-4">
<Sheet <Stack
sx={{ direction="row"
display: 'flex', justifyContent="space-between"
flexDirection: 'row-reverse' alignItems="center"
}} sx={{ marginBottom: '20px' }}
> >
<Breadcrumbs aria-label="breadcrumbs">
<Link
onClick={() => {
router.push('/datastores')
}}
key="Knowledge Space"
underline="hover"
color="neutral"
fontSize="inherit"
>
Knowledge Space
</Link>
<Typography fontSize="inherit">Documents</Typography>
</Breadcrumbs>
<Button <Button
variant="outlined" variant="outlined"
onClick={() => setIsAddDocumentModalShow(true)} onClick={() => setIsAddDocumentModalShow(true)}
sx={{ marginBottom: '20px' }}
> >
+ Add Datasource + Add Datasource
</Button> </Button>
</Sheet> </Stack>
{documents.length ? ( {documents.length ? (
<> <>
<Table <Table
@ -334,7 +353,8 @@ const Documents = () => {
color: activeStep === index ? '#814DDE' : '' color: activeStep === index ? '#814DDE' : ''
}} }}
> >
{item} {index < activeStep ? <CheckCircleOutlinedIcon /> : `${index + 1}.`}
{`${item}`}
</Item> </Item>
))} ))}
</Stack> </Stack>
@ -422,152 +442,224 @@ const Documents = () => {
/> />
</> </>
)} )}
<Typography
component="label"
sx={{
marginTop: '20px'
}}
endDecorator={
<Switch
checked={synchChecked}
onChange={(event: any) =>
setSynchChecked(event.target.checked)
}
/>
}
>
Synch:
</Typography>
</Box> </Box>
<Button <Stack
variant="outlined" direction="row"
onClick={async () => { justifyContent="flex-start"
if (documentName === '') { alignItems="center"
message.error('Please input the name') sx={{ marginBottom: '20px' }}
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
}
}}
> >
Finish <Button
</Button> variant="outlined"
sx={{ marginRight: '20px' }}
onClick={() => setActiveStep(0)}
>
{'< Back'}
</Button>
<Button
variant="outlined"
onClick={async () => {
if (documentName === '') {
message.error('Please input the name')
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
data.success &&
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
data.success &&
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
data.success &&
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
}
}}
>
Finish
</Button>
</Stack>
</> </>
)} )}
</Sheet> </Sheet>

View File

@ -3,6 +3,7 @@
import { useRouter } from 'next/navigation' import { useRouter } from 'next/navigation'
import React, { useState, useEffect } from 'react' import React, { useState, useEffect } from 'react'
import { InboxOutlined } from '@ant-design/icons' import { InboxOutlined } from '@ant-design/icons'
import CheckCircleOutlinedIcon from '@mui/icons-material/CheckCircleOutlined';
import type { UploadProps } from 'antd' import type { UploadProps } from 'antd'
import { message, Upload, Popover } from 'antd' import { message, Upload, Popover } from 'antd'
import { import {
@ -16,6 +17,8 @@ import {
Input, Input,
Textarea, Textarea,
Chip, Chip,
Switch,
Typography,
styled styled
} from '@/lib/mui' } from '@/lib/mui'
import { fetchBaseURL } from '@/app/datastores/constants' import { fetchBaseURL } from '@/app/datastores/constants'
@ -34,9 +37,9 @@ const Item = styled(Sheet)(({ theme }) => ({
})) }))
const stepsOfAddingSpace = [ const stepsOfAddingSpace = [
'1.Knowledge Space Config', 'Knowledge Space Config',
'2.Choose a Datasource type', 'Choose a Datasource type',
'3.Setup the Datasource' 'Setup the Datasource'
] ]
const documentTypeList = [ const documentTypeList = [
{ {
@ -71,6 +74,7 @@ const Index = () => {
const [textSource, setTextSource] = useState<string>('') const [textSource, setTextSource] = useState<string>('')
const [text, setText] = useState<string>('') const [text, setText] = useState<string>('')
const [originFileObj, setOriginFileObj] = useState<any>(null) const [originFileObj, setOriginFileObj] = useState<any>(null)
const [synchChecked, setSynchChecked] = useState<boolean>(true)
const props: UploadProps = { const props: UploadProps = {
name: 'file', name: 'file',
multiple: false, multiple: false,
@ -138,6 +142,9 @@ const Index = () => {
}, },
'& tbody tr: hover a': { '& tbody tr: hover a': {
textDecoration: 'underline' textDecoration: 'underline'
},
'& tbody tr a': {
color: 'rgb(13, 96, 217)'
} }
}} }}
> >
@ -214,9 +221,13 @@ const Index = () => {
{stepsOfAddingSpace.map((item: any, index: number) => ( {stepsOfAddingSpace.map((item: any, index: number) => (
<Item <Item
key={item} key={item}
sx={{ fontWeight: activeStep === index ? 'bold' : '', color: activeStep === index ? '#814DDE' : '' }} sx={{
fontWeight: activeStep === index ? 'bold' : '',
color: activeStep === index ? '#814DDE' : ''
}}
> >
{item} {index < activeStep ? <CheckCircleOutlinedIcon /> : `${index + 1}.`}
{`${item}`}
</Item> </Item>
))} ))}
</Stack> </Stack>
@ -361,95 +372,164 @@ const Index = () => {
/> />
</> </>
)} )}
<Typography
component="label"
sx={{
marginTop: '20px'
}}
endDecorator={
<Switch
checked={synchChecked}
onChange={(event: any) =>
setSynchChecked(event.target.checked)
}
/>
}
>
Synch:
</Typography>
</Box> </Box>
<Button <Stack
variant='outlined' direction="row"
onClick={async () => { justifyContent="flex-start"
if (documentName === '') { alignItems="center"
message.error('Please input the name') sx={{ marginBottom: '20px' }}
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
} else {
message.error(data.err_msg || 'failed')
}
}
}}
> >
Finish <Button
</Button> variant="outlined"
sx={{ marginRight: '20px' }}
onClick={() => setActiveStep(1)}
>
{'< Back'}
</Button>
<Button
variant="outlined"
onClick={async () => {
if (documentName === '') {
message.error('Please input the name')
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
} else {
message.error(data.err_msg || 'failed')
}
}
}}
>
Finish
</Button>
</Stack>
</> </>
)} )}
</Sheet> </Sheet>