haiwen/seahub
1
0
Fork 0
mirror of https://github.com/haiwen/seahub.git synced 2025-09-03 07:55:36 +00:00
Code Issues Releases Wiki Activity

search file use ai (#5690)

Browse Source 
* search file use ai

* Feat: optimize code

* feat: update code

---------

Co-authored-by: er-pai-r <18335219360@163.com>
This commit is contained in:
JoinTyang
2023-10-21 11:38:12 +08:00
committed by GitHub
parent 0a09db3fa4
commit 1a6fe7b9cf
9 changed files with 352 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

263
frontend/src/components/search/search.js
View File

@@ -3,17 +3,30 @@ import PropTypes from 'prop-types';
import isHotkey from 'is-hotkey';
import MediaQuery from 'react-responsive';
import { seafileAPI } from '../../utils/seafile-api';
import { gettext, siteRoot, username } from '../../utils/constants';
import { gettext, siteRoot, username, enableSeafileAI } from '../../utils/constants';
import SearchResultItem from './search-result-item';
import { Utils } from '../../utils/utils';
import { isMac } from '../../utils/extra-attributes';
import toaster from '../toast';
const INDEX_STATE = {
RUNNING: 'running',
UNCREATED: 'uncreated',
FINISHED: 'finished'
};
const SEARCH_MODE = {
SIMILARITY: 'similarity',
NORMAL: 'normal',
};
const propTypes = {
repoID: PropTypes.string,
placeholder: PropTypes.string,
onSearchedClick: PropTypes.func.isRequired,
isPublic: PropTypes.bool,
isLibView: PropTypes.bool,
repoName: PropTypes.string,
};
const PER_PAGE = 10;
@@ -37,7 +50,9 @@ class Search extends Component {
isResultGetted: false,
isCloseShow: false,
isSearchInputShow: false, // for mobile
searchPageUrl: this.baseSearchPageURL
searchPageUrl: this.baseSearchPageURL,
searchMode: SEARCH_MODE.NORMAL,
indexState: INDEX_STATE.UNCREATED,
};
this.inputValue = '';
this.highlightRef = null;
@@ -45,6 +60,8 @@ class Search extends Component {
this.inputRef = React.createRef();
this.searchContainer = React.createRef();
this.searchResultListRef = React.createRef();
this.timer = null;
this.indexStateTimer = null;
}
componentDidMount() {
@@ -53,6 +70,8 @@ class Search extends Component {
componentWillUnmount() {
document.removeEventListener('keydown', this.onDocumentKeydown);
this.indexStateTimer && clearInterval(this.indexStateTimer);
this.timer && clearTimeout(this.timer);
}
onDocumentKeydown = (e) => {
@@ -65,6 +84,7 @@ class Search extends Component {
}
else if (isHotkey('esc', e)) {
e.preventDefault();
this.inputRef && this.inputRef.current && this.inputRef.current.blur();
this.resetToDefault();
} else if (isHotkey('enter', e)) {
this.onEnter(e);
@@ -76,10 +96,20 @@ class Search extends Component {
};
onFocusHandler = () => {
this.setState({
width: '570px',
isMaskShow: true,
isCloseShow: true
const { searchMode, indexState: currentIndexState } = this.state;
const { repoID } = this.props;
this.setState({ width: '570px', isMaskShow: true, isCloseShow: true }, () => {
if (searchMode !== SEARCH_MODE.SIMILARITY) return;
if (currentIndexState === INDEX_STATE.FINISHED) return;
seafileAPI.queryLibraryIndexState(repoID).then(res => {
const { state: indexState, task_id: taskId } = res.data;
this.setState({ indexState }, () => {
if (indexState !== INDEX_STATE.RUNNING) return;
this.queryIndexTaskStatus(taskId);
});
}).catch(error => {
this.setState({ indexState: INDEX_STATE.UNCREATED });
});
});
};
@@ -137,38 +167,39 @@ class Search extends Component {
};
onChangeHandler = (event) => {
let _this = this;
this.setState({value: event.target.value});
let newValue = event.target.value;
if (this.inputValue === newValue.trim()) {
return false;
}
const newValue = event.target.value;
this.setState({ value: newValue }, () => {
if (this.inputValue === newValue.trim()) return;
this.inputValue = newValue.trim();
this.onSearch();
});
};
if (this.inputValue === '' || _this.getValueLength(this.inputValue) < 3) {
onSearch = () => {
const { value } = this.state;
const { repoID } = this.props;
const _this = this;
this.timer && clearTimeout(this.timer);
if (_this.inputValue === '' || _this.getValueLength(_this.inputValue) < 3) {
this.setState({
highlightIndex: 0,
resultItems: [],
isResultShow: false,
isResultGetted: false
});
return false;
return;
}
let repoID = this.props.repoID;
let queryData = {
q: newValue,
const queryData = {
q: value,
search_repo: repoID ? repoID : 'all',
search_ftypes: 'all',
};
if (this.timer) {
clearTimeout(this.timer);
}
this.timer = setTimeout(_this.getSearchResult(queryData), 500);
};
getSearchResult(queryData) {
getSearchResult = (queryData) => {
if (this.source) {
this.source.cancel('prev request is cancelled');
}
@@ -180,7 +211,7 @@ class Search extends Component {
});
this.source = seafileAPI.getSource();
this.sendRequest(queryData, this.source.token, 1);
}
};
sendRequest = (queryData, cancelToken, page) => {
let isPublic = this.props.isPublic;
@@ -215,6 +246,15 @@ class Search extends Component {
this.updateSearchPageURL(queryData);
queryData['per_page'] = PER_PAGE;
queryData['page'] = page;
if (this.state.searchMode === SEARCH_MODE.NORMAL) {
this.onNormalSearch(queryData, cancelToken, page);
} else {
this.onSimilaritySearch(queryData, cancelToken, page);
}
}
};
onNormalSearch = (queryData, cancelToken, page) => {
seafileAPI.searchFiles(queryData, cancelToken).then(res => {
this.source = null;
if (res.data.total > 0) {
@@ -225,7 +265,8 @@ class Search extends Component {
page: page + 1,
hasMore: res.data.has_more,
});
} else {
return;
}
this.setState({
highlightIndex: 0,
resultItems: [],
@@ -233,14 +274,47 @@ class Search extends Component {
isResultGetted: true,
hasMore: res.data.has_more,
});
}
}).catch(error => {
/* eslint-disable */
console.log(error);
/* eslint-enable */
this.setState({ isLoading: false });
});
};
onSimilaritySearch = (queryData, cancelToken, page) => {
const { indexState } = this.state;
if (indexState === INDEX_STATE.UNCREATED) {
toaster.warning(gettext('Please create index first.'));
return;
}
if (indexState === INDEX_STATE.RUNNING) {
toaster.warning(gettext('Indexing, please try again later.'));
return;
}
seafileAPI.similaritySearchFiles(queryData, cancelToken).then(res => {
this.source = null;
if (res.data && res.data.children_list.length > 0) {
this.setState({
resultItems: [...this.state.resultItems, ...this.formatSimilarityItems(res.data.children_list)],
isResultGetted: true,
isLoading: false,
page: page + 1,
hasMore: res.data.has_more,
});
return;
}
this.setState({
highlightIndex: 0,
resultItems: [],
isLoading: false,
isResultGetted: true,
hasMore: res.data.has_more,
});
}).catch(error => {
/* eslint-disable */
console.log(error);
this.setState({ isLoading: false });
});
};
onResultListScroll = (e) => {
@@ -299,8 +373,26 @@ class Search extends Component {
return items;
}
formatSimilarityItems(data) {
let items = [];
let repo_id = this.props.repoID;
for (let i = 0; i < data.length; i++) {
items[i] = {};
items[i]['index'] = [i];
items[i]['name'] = data[i].path.substring(data[i].path.lastIndexOf('/')+1);
items[i]['path'] = data[i].path;
items[i]['repo_id'] = repo_id;
items[i]['repo_name'] = this.props.repoName;
items[i]['is_dir'] = false;
items[i]['link_content'] = decodeURI(data[i].path).substring(1);
items[i]['content'] = data[i].sentence;
items[i]['thumbnail_url'] = '';
}
return items;
}
resetToDefault() {
this.inputValue = null;
this.inputValue = '';
this.setState({
width: '',
value: '',
@@ -315,10 +407,25 @@ class Search extends Component {
}
renderSearchResult() {
const { resultItems, highlightIndex } = this.state;
if (!this.state.isResultShow) {
return;
const { resultItems, highlightIndex, indexState, searchMode, width } = this.state;
if (!width) return null;
if (searchMode === SEARCH_MODE.SIMILARITY && indexState === INDEX_STATE.UNCREATED) {
return (
<div className="search-mode-similarity-index-status index-status-uncreated" onClick={this.onCreateIndex}>
{gettext('Click create index')}
</div>
);
}
if (searchMode === SEARCH_MODE.SIMILARITY && indexState === INDEX_STATE.RUNNING) {
return (
<div className="search-mode-similarity-index-status">
{gettext('Indexing...')}
</div>
);
}
if (!this.state.isResultShow) return null;
if (!this.state.isResultGetted || this.getValueLength(this.inputValue) < 3) {
return (
<span className="loading-icon loading-tip"></span>
@@ -329,7 +436,8 @@ class Search extends Component {
<div className="search-result-none">{gettext('No results matching.')}</div>
);
}
return (
const results = (
<ul className="search-result-list" ref={this.searchResultListRef}>
{resultItems.map((item, index) => {
const isHighlight = index === highlightIndex;
@@ -345,6 +453,17 @@ class Search extends Component {
})}
</ul>
);
return (
<>
<MediaQuery query="(min-width: 768px)">
<div className="search-result-list-container">{results}</div>
</MediaQuery>
<MediaQuery query="(max-width: 767.8px)">
{results}
</MediaQuery>
</>
);
}
onSearchToggle = () => {
@@ -354,10 +473,81 @@ class Search extends Component {
});
};
onChangeSearchMode = (event) => {
const searchMode = event.target.getAttribute('mode-type');
if (searchMode === this.state.searchMode) return;
const { repoID } = this.props;
const { indexState: currentIndexState } = this.state;
this.timer && clearTimeout(this.timer);
this.setState({ searchMode }, () => {
if (searchMode === SEARCH_MODE.NORMAL) {
this.onSearch();
this.indexStateTimer && clearInterval(this.indexStateTimer);
return;
}
if (searchMode === SEARCH_MODE.SIMILARITY) {
if (currentIndexState === INDEX_STATE.FINISHED) {
this.onSearch();
return;
}
seafileAPI.queryLibraryIndexState(repoID).then(res => {
const { state: indexState, task_id: taskId } = res.data;
this.setState({ indexState }, () => {
if (indexState === INDEX_STATE.FINISHED) {
this.onSearch();
return;
}
if (indexState === INDEX_STATE.RUNNING) {
this.queryIndexTaskStatus(taskId, this.onSearch);
return;
}
});
}).catch(error => {
this.setState({ indexState: INDEX_STATE.UNCREATED });
});
}
});
};
queryIndexTaskStatus = (taskId, callback) => {
if (!taskId) return;
this.indexStateTimer = setInterval(() => {
seafileAPI.queryIndexTaskStatus(taskId).then(res => {
const isFinished = res.data.is_finished;
if (isFinished) {
this.setState({ indexState: INDEX_STATE.FINISHED }, () => {
callback && callback();
});
this.indexStateTimer && clearInterval(this.indexStateTimer);
this.indexStateTimer = null;
}
}).catch(error => {
this.indexStateTimer && clearInterval(this.indexStateTimer);
this.indexStateTimer = null;
const errorMsg = Utils.getErrorMsg(error);
toaster.danger(errorMsg);
this.setState({ indexState: INDEX_STATE.UNCREATED });
});
}, 3000);
};
onCreateIndex = () => {
this.setState({ indexState: INDEX_STATE.RUNNING });
seafileAPI.createLibraryIndex(this.props.repoID).then(res => {
const taskId = res.data.task_id;
this.queryIndexTaskStatus(taskId);
}).catch(error => {
const errorMsg = Utils.getErrorMsg(error);
toaster.danger(errorMsg);
this.setState({ indexState: INDEX_STATE.UNCREATED });
});
};
render() {
let width = this.state.width !== 'default' ? this.state.width : '';
let style = {'width': width};
const { searchPageUrl, isMaskShow } = this.state;
const { searchPageUrl, isMaskShow, searchMode, indexState, isCloseShow } = this.state;
const placeholder = `${this.props.placeholder}${isMaskShow ? '' : ` (${controlKey} + f )`}`;
return (
<Fragment>
@@ -378,6 +568,7 @@ class Search extends Component {
onChange={this.onChangeHandler}
autoComplete="off"
ref={this.inputRef}
readOnly={isCloseShow && enableSeafileAI && SEARCH_MODE.SIMILARITY === searchMode && indexState !== INDEX_STATE.FINISHED}
/>
{(this.state.isCloseShow && username) &&
<a href={searchPageUrl} className="search-icon-right input-icon-addon fas fa-external-link-alt search-icon-arrow"></a>
@@ -391,6 +582,12 @@ class Search extends Component {
onScroll={this.onResultListScroll}
ref={this.searchContainer}
>
{isCloseShow && enableSeafileAI && this.props.isLibView &&
<div className="search-mode-container">
<div className={`search-mode-item ${SEARCH_MODE.NORMAL === searchMode ? 'search-mode-active' : ''}`} mode-type={SEARCH_MODE.NORMAL} onClick={this.onChangeSearchMode}>{gettext('Normal search')}</div>
<div className={`search-mode-item ${SEARCH_MODE.SIMILARITY === searchMode ? 'search-mode-active' : ''}`} mode-type={SEARCH_MODE.SIMILARITY} onClick={this.onChangeSearchMode}>{gettext('Similarity search')}</div>
</div>
}
{this.renderSearchResult()}
</div>
</div>

2
frontend/src/components/toolbar/common-toolbar.js
View File

@@ -26,6 +26,8 @@ class CommonToolbar extends React.Component {
repoID={repoID}
placeholder={this.props.searchPlaceholder || gettext('Search files')}
onSearchedClick={this.props.onSearchedClick}
isLibView={this.props.isLibView}
repoName={repoName}
/>
)}
{this.props.isLibView && !isPro &&

46
frontend/src/css/search.css
View File

@@ -79,6 +79,8 @@
position: relative;
top: 0;
box-shadow: none;
display: flex;
flex-direction: column;
}
.search-result-container .search-result-none {
@@ -92,6 +94,12 @@
list-style: none;
}
.search-result-container .search-result-list-container {
overflow: auto;
flex: 1;
}
.search-result-list .item-content .item-name {
color: #eb8205!important;
}
@@ -300,3 +308,41 @@
width: 100%;
}
}
.search-result-container .search-mode-container {
display: flex;
align-items: center;
height: 40px;
border-bottom: 1px solid #ddd;
padding: 0 12px;
margin-right: 16px;
}
.search-result-container .search-mode-item {
display: flex;
align-items: center;
height: 40px;
line-height: 40px;
margin-right: 30px;
font-size: 14px;
cursor: pointer;
border-bottom: 2px solid transparent;
user-select: none;
}
.search-result-container .search-mode-item.search-mode-active {
color: #ff8001;
border-bottom-color: #ff8001;
}
.search-result-container .search-mode-similarity-index-status {
height: 64px;
display: flex;
flex-direction: column;
align-items: center;
justify-content: center;
}
.search-result-container .search-mode-similarity-index-status.index-status-uncreated {
cursor: pointer;
}

3
frontend/src/utils/constants.js
View File

@@ -94,6 +94,9 @@ export const enableVideoThumbnail = window.app.pageOptions.enableVideoThumbnail;
export const enableOnlyoffice = window.app.pageOptions.enableOnlyoffice || false;
export const onlyofficeConverterExtensions = window.app.pageOptions.onlyofficeConverterExtensions || [];
// seafile_ai
export const enableSeafileAI = window.app.pageOptions.enableSeafileAI || false;
// dtable
export const workspaceID = window.app.pageOptions.workspaceID;
export const showLogoutIcon = window.app.pageOptions.showLogoutIcon;

26
seahub/ai/apis.py
View File

@@ -15,7 +15,8 @@ from seahub.api2.utils import api_error
from seahub.views import check_folder_permission
from seahub.utils.repo import parse_repo_perm
from seahub.ai.utils import create_library_sdoc_index, get_dir_file_recursively, similarity_search_in_library, \
update_library_sdoc_index, delete_library_index, query_task_status, get_dir_sdoc_info_list
update_library_sdoc_index, delete_library_index, query_task_status, get_dir_sdoc_info_list, \
query_library_index_state
from seaserv import seafile_api
@@ -218,6 +219,29 @@ class TaskStatus(APIView):
return Response(resp_json, resp.status_code)
class LibraryIndexState(APIView):
authentication_classes = (TokenAuthentication, SessionAuthentication)
permission_classes = (IsAuthenticated, )
throttle_classes = (UserRateThrottle, )
def get(self, request):
repo_id = request.GET.get('repo_id')
if not repo_id:
return api_error(status.HTTP_400_BAD_REQUEST, 'repo_id invalid')
try:
resp = query_library_index_state(repo_id)
if resp.status_code == 500:
logger.error('query library index state error status: %s body: %s', resp.status_code, resp.text)
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Internal Server Error')
resp_json = resp.json()
except Exception as e:
logger.error(e)
return api_error(status.HTTP_500_INTERNAL_SERVER_ERROR, 'Internal Server Error')
return Response(resp_json, resp.status_code)
class RepoFiles(APIView):
authentication_classes = (SeafileAiAuthentication, )
throttle_classes = (UserRateThrottle, )

7
seahub/ai/utils.py
View File

@@ -100,3 +100,10 @@ def query_task_status(task_id):
url = urljoin(SEAFILE_AI_SERVER_URL, '/api/v1/task-status/')
resp = requests.get(url, headers=headers, params={'task_id': task_id})
return resp
def query_library_index_state(associate_id):
headers = gen_headers()
url = urljoin(SEAFILE_AI_SERVER_URL, '/api/v1/library-index-state/')
resp = requests.get(url, headers=headers, params={'associate_id': associate_id})
return resp

5
seahub/base/context_processors.py
View File

@@ -24,7 +24,7 @@ from seahub.settings import SEAFILE_VERSION, SITE_DESCRIPTION, \
MEDIA_ROOT, SHOW_LOGOUT_ICON, CUSTOM_LOGO_PATH, CUSTOM_FAVICON_PATH, \
ENABLE_SEAFILE_DOCS, LOGIN_BG_IMAGE_PATH, \
CUSTOM_LOGIN_BG_PATH, ENABLE_SHARE_LINK_REPORT_ABUSE, \
PRIVACY_POLICY_LINK, TERMS_OF_SERVICE_LINK, ENABLE_SEADOC
PRIVACY_POLICY_LINK, TERMS_OF_SERVICE_LINK, ENABLE_SEADOC, ENABLE_SEAFILE_AI
from seahub.organizations.models import OrgAdminSettings
from seahub.organizations.settings import ORG_ENABLE_ADMIN_CUSTOM_LOGO
@@ -165,7 +165,8 @@ def base(request):
'side_nav_footer_custom_html': SIDE_NAV_FOOTER_CUSTOM_HTML,
'about_dialog_custom_html': ABOUT_DIALOG_CUSTOM_HTML,
'enable_repo_auto_del': ENABLE_REPO_AUTO_DEL,
'enable_seadoc': ENABLE_SEADOC
'enable_seadoc': ENABLE_SEADOC,
'enable_seafile_ai': ENABLE_SEAFILE_AI,
}
if request.user.is_staff:

1
seahub/templates/base_for_react.html
View File

@@ -145,6 +145,7 @@
enableOnlyoffice: {% if enableOnlyoffice %} true {% else %} false {% endif %},
onlyofficeConverterExtensions: {% if onlyofficeConverterExtensions %} {{onlyofficeConverterExtensions|safe}} {% else %} null {% endif %},
enableSeadoc: {% if enable_seadoc %} true {% else %} false {% endif %},
enableSeafileAI: {% if enable_seafile_ai %} true {% else %} false {% endif %},
}
};
</script>

20
seahub/urls.py
View File

@@ -201,7 +201,8 @@ from seahub.seadoc.views import sdoc_revision, sdoc_revisions
from seahub.ocm.settings import OCM_ENDPOINT
from seahub.ai.apis import LibrarySdocIndexes, SimilaritySearchInLibrary, LibrarySdocIndex, RepoFiles, TaskStatus
from seahub.ai.apis import LibrarySdocIndexes, SimilaritySearchInLibrary, LibrarySdocIndex, RepoFiles, TaskStatus, \
LibraryIndexState
urlpatterns = [
path('accounts/', include('seahub.base.registration_urls')),
@@ -535,13 +536,6 @@ urlpatterns = [
re_path(r'api/v2.1/ocm/providers/(?P<provider_id>[-0-9a-f]{36})/repos/(?P<repo_id>[-0-9a-f]{36})/download-link/$', OCMReposDownloadLinkView.as_view(), name='api-v2.1-ocm-repos-download-link'),
re_path(r'api/v2.1/ocm/providers/(?P<provider_id>[-0-9a-f]{36})/repos/(?P<repo_id>[-0-9a-f]{36})/upload-link/$', OCMReposUploadLinkView.as_view(), name='api-v2.1-ocm-repos-upload-link'),
# seafile-ai
re_path(r'^api/v2.1/ai/library-sdoc-indexes/$', LibrarySdocIndexes.as_view(), name='api-v2.1-ai-library-sdoc-indexes'),
re_path(r'^api/v2.1/ai/similarity-search-in-library/$', SimilaritySearchInLibrary.as_view(), name='api-v2.1-ai-similarity-search-in-library'),
re_path(r'^api/v2.1/ai/library-sdoc-index/$', LibrarySdocIndex.as_view(), name='api-v2.1-ai-library-sdoc-index'),
re_path(r'^api/v2.1/ai/repo/files/$', RepoFiles.as_view(), name='api-v2.1-ai-repo-files'),
re_path(r'^api/v2.1/ai/task-status/$', TaskStatus.as_view(), name='api-v2.1-ai-task-status'),
# admin: activities
re_path(r'^api/v2.1/admin/user-activities/$', UserActivitiesView.as_view(), name='api-v2.1-admin-user-activity'),
@@ -964,3 +958,13 @@ if getattr(settings, 'ENABLE_SEADOC', False):
urlpatterns += [
re_path(r'^api/v2.1/seadoc/', include('seahub.seadoc.urls')),
]
if settings.ENABLE_SEAFILE_AI:
urlpatterns += [
re_path(r'^api/v2.1/ai/library-sdoc-indexes/$', LibrarySdocIndexes.as_view(), name='api-v2.1-ai-library-sdoc-indexes'),
re_path(r'^api/v2.1/ai/similarity-search-in-library/$', SimilaritySearchInLibrary.as_view(), name='api-v2.1-ai-similarity-search-in-library'),
re_path(r'^api/v2.1/ai/library-sdoc-index/$', LibrarySdocIndex.as_view(), name='api-v2.1-ai-library-sdoc-index'),
re_path(r'^api/v2.1/ai/repo/files/$', RepoFiles.as_view(), name='api-v2.1-ai-repo-files'),
re_path(r'^api/v2.1/ai/task-status/$', TaskStatus.as_view(), name='api-v2.1-ai-task-status'),
re_path(r'^api/v2.1/ai/library-index-state/$', LibraryIndexState.as_view(), name='api-v2.1-ai-library-index-state'),
]