Merge remote-tracking branch 'origin/new-page-framework' into llm_framework

This commit is contained in:
aries_ckt 2023-06-30 16:36:27 +08:00
commit 0a382b64db
3 changed files with 553 additions and 335 deletions

View File

@ -1,13 +1,21 @@
'use client'
import { useSearchParams } from 'next/navigation'
import { useSearchParams, useRouter } from 'next/navigation'
import React, { useState, useEffect } from 'react'
import { useColorScheme, Table, Stack } from '@/lib/mui'
import {
useColorScheme,
Table,
Stack,
Typography,
Breadcrumbs,
Link
} from '@/lib/mui'
import { Popover, Pagination } from 'antd'
import { fetchBaseURL } from '@/app/datastores/constants'
const page_size = 20
const ChunkList = () => {
const router = useRouter()
const { mode } = useColorScheme()
const spaceName = useSearchParams().get('spacename')
const documentId = useSearchParams().get('documentid')
@ -41,99 +49,137 @@ const ChunkList = () => {
}, [])
return (
<div className="p-4">
{chunkList.length ? (
<>
<Table
color="primary"
variant="plain"
size="lg"
sx={{
'& tbody tr: hover': {
backgroundColor:
mode === 'light' ? 'rgb(246, 246, 246)' : 'rgb(33, 33, 40)'
},
'& tbody tr: hover a': {
textDecoration: 'underline'
}
<Stack
direction="row"
justifyContent="flex-start"
alignItems="center"
sx={{ marginBottom: '20px' }}
>
<Breadcrumbs aria-label="breadcrumbs">
<Link
onClick={() => {
router.push('/datastores')
}}
key="Knowledge Space"
underline="hover"
color="neutral"
fontSize="inherit"
>
<thead>
<tr>
<th>Name</th>
<th>Content</th>
<th>Meta Data</th>
</tr>
</thead>
<tbody>
{chunkList.map((row: any) => (
<tr key={row.id}>
<td>{row.doc_name}</td>
<td>
{
<Popover content={row.content} trigger="hover">
{row.content.length > 10
? `${row.content.slice(0, 10)}...`
: row.content}
</Popover>
}
</td>
<td>
{
<Popover
content={JSON.stringify(row.meta_info || '{}', null, 2)}
trigger="hover"
>
{row.meta_info.length > 10
? `${row.meta_info.slice(0, 10)}...`
: row.meta_info}
</Popover>
}
</td>
</tr>
))}
</tbody>
</Table>
<Stack
direction="row"
justifyContent="flex-end"
sx={{
marginTop: '20px'
Knowledge Space
</Link>
<Link
onClick={() => {
router.push(`/datastores/documents?name=${spaceName}`)
}}
key="Knowledge Space"
underline="hover"
color="neutral"
fontSize="inherit"
>
<Pagination
defaultPageSize={20}
showSizeChanger={false}
current={current}
total={total}
onChange={async (page) => {
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/chunk/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
document_id: documentId,
page,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setChunkList(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
Documents
</Link>
<Typography fontSize="inherit">Chunks</Typography>
</Breadcrumbs>
</Stack>
<div className="p-4">
{chunkList.length ? (
<>
<Table
color="primary"
variant="plain"
size="lg"
sx={{
'& tbody tr: hover': {
backgroundColor:
mode === 'light' ? 'rgb(246, 246, 246)' : 'rgb(33, 33, 40)'
},
'& tbody tr: hover a': {
textDecoration: 'underline'
}
}}
hideOnSinglePage
/>
</Stack>
</>
) : (
<></>
)}
>
<thead>
<tr>
<th>Name</th>
<th>Content</th>
<th>Meta Data</th>
</tr>
</thead>
<tbody>
{chunkList.map((row: any) => (
<tr key={row.id}>
<td>{row.doc_name}</td>
<td>
{
<Popover content={row.content} trigger="hover">
{row.content.length > 10
? `${row.content.slice(0, 10)}...`
: row.content}
</Popover>
}
</td>
<td>
{
<Popover
content={JSON.stringify(
row.meta_info || '{}',
null,
2
)}
trigger="hover"
>
{row.meta_info.length > 10
? `${row.meta_info.slice(0, 10)}...`
: row.meta_info}
</Popover>
}
</td>
</tr>
))}
</tbody>
</Table>
<Stack
direction="row"
justifyContent="flex-end"
sx={{
marginTop: '20px'
}}
>
<Pagination
defaultPageSize={20}
showSizeChanger={false}
current={current}
total={total}
onChange={async (page) => {
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/chunk/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
document_id: documentId,
page,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setChunkList(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
}}
hideOnSinglePage
/>
</Stack>
</>
) : (
<></>
)}
</div>
</div>
)
}

View File

@ -13,10 +13,15 @@ import {
Input,
Textarea,
Chip,
Switch,
Typography,
Breadcrumbs,
Link,
styled
} from '@/lib/mui'
import moment from 'moment'
import { InboxOutlined } from '@ant-design/icons'
import CheckCircleOutlinedIcon from '@mui/icons-material/CheckCircleOutlined';
import type { UploadProps } from 'antd'
import { Upload, Pagination, Popover, message } from 'antd'
import { fetchBaseURL } from '@/app/datastores/constants'
@ -33,8 +38,8 @@ const Item = styled(Sheet)(({ theme }) => ({
color: theme.vars.palette.text.secondary
}))
const stepsOfAddingDocument = [
'1.Choose a Datasource type',
'2.Setup the Datasource'
'Choose a Datasource type',
'Setup the Datasource'
]
const documentTypeList = [
{
@ -72,6 +77,7 @@ const Documents = () => {
const [originFileObj, setOriginFileObj] = useState<any>(null)
const [total, setTotal] = useState<number>(0)
const [current, setCurrent] = useState<number>(0)
const [synchChecked, setSynchChecked] = useState<boolean>(true)
const props: UploadProps = {
name: 'file',
multiple: false,
@ -112,20 +118,33 @@ const Documents = () => {
}, [])
return (
<div className="p-4">
<Sheet
sx={{
display: 'flex',
flexDirection: 'row-reverse'
}}
<Stack
direction="row"
justifyContent="space-between"
alignItems="center"
sx={{ marginBottom: '20px' }}
>
<Breadcrumbs aria-label="breadcrumbs">
<Link
onClick={() => {
router.push('/datastores')
}}
key="Knowledge Space"
underline="hover"
color="neutral"
fontSize="inherit"
>
Knowledge Space
</Link>
<Typography fontSize="inherit">Documents</Typography>
</Breadcrumbs>
<Button
variant="outlined"
onClick={() => setIsAddDocumentModalShow(true)}
sx={{ marginBottom: '20px' }}
>
+ Add Datasource
</Button>
</Sheet>
</Stack>
{documents.length ? (
<>
<Table
@ -334,7 +353,8 @@ const Documents = () => {
color: activeStep === index ? '#814DDE' : ''
}}
>
{item}
{index < activeStep ? <CheckCircleOutlinedIcon /> : `${index + 1}.`}
{`${item}`}
</Item>
))}
</Stack>
@ -422,152 +442,224 @@ const Documents = () => {
/>
</>
)}
<Typography
component="label"
sx={{
marginTop: '20px'
}}
endDecorator={
<Switch
checked={synchChecked}
onChange={(event: any) =>
setSynchChecked(event.target.checked)
}
/>
}
>
Synch:
</Typography>
</Box>
<Button
variant="outlined"
onClick={async () => {
if (documentName === '') {
message.error('Please input the name')
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
}
}}
<Stack
direction="row"
justifyContent="flex-start"
alignItems="center"
sx={{ marginBottom: '20px' }}
>
Finish
</Button>
<Button
variant="outlined"
sx={{ marginRight: '20px' }}
onClick={() => setActiveStep(0)}
>
{'< Back'}
</Button>
<Button
variant="outlined"
onClick={async () => {
if (documentName === '') {
message.error('Please input the name')
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
data.success &&
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
data.success &&
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
data.success &&
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
if (data.success) {
message.success('success')
setIsAddDocumentModalShow(false)
const res = await fetch(
`${fetchBaseURL}/knowledge/${spaceName}/document/list`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
page: current,
page_size
})
}
)
const data = await res.json()
if (data.success) {
setDocuments(data.data.data)
setTotal(data.data.total)
setCurrent(data.data.page)
}
} else {
message.error(data.err_msg || 'failed')
}
}
}}
>
Finish
</Button>
</Stack>
</>
)}
</Sheet>

View File

@ -3,6 +3,7 @@
import { useRouter } from 'next/navigation'
import React, { useState, useEffect } from 'react'
import { InboxOutlined } from '@ant-design/icons'
import CheckCircleOutlinedIcon from '@mui/icons-material/CheckCircleOutlined';
import type { UploadProps } from 'antd'
import { message, Upload, Popover } from 'antd'
import {
@ -16,6 +17,8 @@ import {
Input,
Textarea,
Chip,
Switch,
Typography,
styled
} from '@/lib/mui'
import { fetchBaseURL } from '@/app/datastores/constants'
@ -34,9 +37,9 @@ const Item = styled(Sheet)(({ theme }) => ({
}))
const stepsOfAddingSpace = [
'1.Knowledge Space Config',
'2.Choose a Datasource type',
'3.Setup the Datasource'
'Knowledge Space Config',
'Choose a Datasource type',
'Setup the Datasource'
]
const documentTypeList = [
{
@ -71,6 +74,7 @@ const Index = () => {
const [textSource, setTextSource] = useState<string>('')
const [text, setText] = useState<string>('')
const [originFileObj, setOriginFileObj] = useState<any>(null)
const [synchChecked, setSynchChecked] = useState<boolean>(true)
const props: UploadProps = {
name: 'file',
multiple: false,
@ -138,6 +142,9 @@ const Index = () => {
},
'& tbody tr: hover a': {
textDecoration: 'underline'
},
'& tbody tr a': {
color: 'rgb(13, 96, 217)'
}
}}
>
@ -214,9 +221,13 @@ const Index = () => {
{stepsOfAddingSpace.map((item: any, index: number) => (
<Item
key={item}
sx={{ fontWeight: activeStep === index ? 'bold' : '', color: activeStep === index ? '#814DDE' : '' }}
sx={{
fontWeight: activeStep === index ? 'bold' : '',
color: activeStep === index ? '#814DDE' : ''
}}
>
{item}
{index < activeStep ? <CheckCircleOutlinedIcon /> : `${index + 1}.`}
{`${item}`}
</Item>
))}
</Stack>
@ -361,95 +372,164 @@ const Index = () => {
/>
</>
)}
<Typography
component="label"
sx={{
marginTop: '20px'
}}
endDecorator={
<Switch
checked={synchChecked}
onChange={(event: any) =>
setSynchChecked(event.target.checked)
}
/>
}
>
Synch:
</Typography>
</Box>
<Button
variant='outlined'
onClick={async () => {
if (documentName === '') {
message.error('Please input the name')
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
} else {
message.error(data.err_msg || 'failed')
}
}
}}
<Stack
direction="row"
justifyContent="flex-start"
alignItems="center"
sx={{ marginBottom: '20px' }}
>
Finish
</Button>
<Button
variant="outlined"
sx={{ marginRight: '20px' }}
onClick={() => setActiveStep(1)}
>
{'< Back'}
</Button>
<Button
variant="outlined"
onClick={async () => {
if (documentName === '') {
message.error('Please input the name')
return
}
if (documentType === 'webPage') {
if (webPageUrl === '') {
message.error('Please input the Web Page URL')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
content: webPageUrl,
doc_type: 'URL'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
} else {
message.error(data.err_msg || 'failed')
}
} else if (documentType === 'file') {
if (!originFileObj) {
message.error('Please select a file')
return
}
const formData = new FormData()
formData.append('doc_name', documentName)
formData.append('doc_file', originFileObj)
formData.append('doc_type', 'DOCUMENT')
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/upload`,
{
method: 'POST',
body: formData
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
} else {
message.error(data.err_msg || 'failed')
}
} else {
if (text === '') {
message.error('Please input the text')
return
}
const res = await fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/add`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_name: documentName,
source: textSource,
content: text,
doc_type: 'TEXT'
})
}
)
const data = await res.json()
if (data.success) {
message.success('success')
setIsAddKnowledgeSpaceModalShow(false)
synchChecked &&
fetch(
`${fetchBaseURL}/knowledge/${knowledgeSpaceName}/document/sync`,
{
method: 'POST',
headers: {
'Content-Type': 'application/json'
},
body: JSON.stringify({
doc_ids: [data.data]
})
}
)
} else {
message.error(data.err_msg || 'failed')
}
}
}}
>
Finish
</Button>
</Stack>
</>
)}
</Sheet>