Fetch chunk for matches

I have this module for semantic search through the contents of text files:

// AISearchByFileContent.js

import { t } from "../language/i18n";
import threadManagerInstance from "./ThreadManager";
import { searchTextByBatch } from "./SemanticSearch";
import { FileType } from './AppFile';
import { fetch } from "@react-native-community/netinfo";
import { deleteFromBase, executeQuery } from './DBManager';
import { getGDriveFileContentAndModifiedTimeOrDeleteFromBase, MAX_CHUNK_LENGTH, MIN_CHUNK_LENGTH } from './constantsAndFunctions';
import { SearchResultTypes } from '../screens/multifileSearch';
import ProgressBar from './GUI_Elements/ProgressBar';
import TextEmbenderInstance from './AI_Modules/TextEmbender';
import { gdriveFileManager, localFileManager } from './file_manager/FileManagerFactory';
import Toast from 'react-native-toast-message';

const setProgresses = ({ mainProgress, progressText, secondaryProgress, closeHandler }) => {

mainProgress !== undefined && ProgressBar.setProgress(mainProgress);
progressText !== undefined && ProgressBar.setText(progressText);
secondaryProgress !== undefined && ProgressBar.setSecondaryProgress(secondaryProgress);
closeHandler !== undefined && ProgressBar.setCloseHandler(closeHandler);

}

// Helper function to get file content based on file type
async function getFileContent(file, gDriveFileDetails = null) {
if (file.fileType === FileType.G_DRIVE_FILE) {
const {content, modificationTime} = await getGDriveFileContentAndModifiedTimeOrDeleteFromBase(
file.fileId,
gDriveFileDetails[file.fileId]?.modifiedTime
);

file.modificationTime = modificationTime;
return content;
} else {
file.content = await localFileManager.getFileContent(file.fileId);
return file.content;
}
}

export async function AISearchByFileContent(
query, hasPermission, setSearchList, setRerender,
currentSearchResultType, userRef, threadId
) {

const SMALL_FILE_SIZE = 2300;

// Set the current search result type to semantic
// Assuming you have a state or context to manage this
currentSearchResultType.current = SearchResultTypes.AI;

setSearchList([]);

let startTime = performance.now();

setProgresses({
mainProgress: 0,
progressText: t('Analyzing available files'),
closeHandler: ()=>{
threadManagerInstance.clearCurrentThreadId();
ProgressBar.setText(t('Stopping...'));
}
});

try {
// Fetch all files from the database
const allFilesRows = await executeQuery(
'SELECT fileId, isAiSearchable, fileType, lastOpening, relativeFilePath, email, modifiedTimeBeforeIndexing FROM files ORDER BY lastOpening DESC'
);

const gDriveFiles = [];
const bigFileFiles = [];
const smallFileFiles = [];

// Categorize files
for (let i = 0; i < allFilesRows.length; i++) {
const item = allFilesRows.item(i);
const fileId = item.fileId;

if (item.fileType !== FileType.G_DRIVE_FILE) {
// If we don't have permission and file is SHARED_FILE, skip reading
if (!hasPermission && item.fileType === FileType.SHARED_FILE) {
continue;
}

try {
if (await localFileManager.fileExists(fileId)) {

const fileDitails = await localFileManager.getFileDetails(fileId);
item.modificationTime = fileDitails.modifiedTime;

if (fileDitails.size < SMALL_FILE_SIZE) {
smallFileFiles.push(item);
} else {
bigFileFiles.push(item);
}
} else {
await deleteFromBase(fileId);
}
} catch (error) {
console.error(`Error accessing file ${fileId}:`, error);
continue;
}
} else {
if (userRef.current !== null && userRef.current.email === item.email) {
gDriveFiles.push(item);
}
}
}

// Calculate total steps for progress
let totalSteps = smallFileFiles.length + bigFileFiles.length + gDriveFiles.length;


if( totalSteps === 0 ) return;


const queryEmbedding = await TextEmbenderInstance.getSentenceEmbedding(query);

let completedSteps = 0;

// Function to update progress
const updateProgress = () => {

if(performance.now()-startTime > 1000) {
startTime = performance.now();
const progress = ((completedSteps-1) / totalSteps) * 100;
setProgresses({
mainProgress: progress,
progressText: `${t('Search in file')} ${completedSteps} / ${totalSteps}`,
secondaryProgress: 0
});
}
};

const secondaryProgressCallback = (secondaryProgress, progressText) => {
if(performance.now()-startTime > 1000) {
startTime = performance.now();
setProgresses({ progressText: `${completedSteps} / ${totalSteps} | ${progressText}`, secondaryProgress });
}
}

// Collect best matches from each file
const bestMatches = [];

// Function to process a list of files
const processFiles = async (files, gDriveFileDetails = null) => {
for (let i = 0; i < files.length; i++) {
if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

completedSteps += 1;
updateProgress();

const file = files[i];

try {
file.content = await getFileContent(file, gDriveFileDetails);
let secProgressCallback = undefined;
if(file.modificationTime !== file.modifiedTimeBeforeIndexing) {
secProgressCallback = secondaryProgressCallback;

const progress = (completedSteps / totalSteps) * 100;
setProgresses({
mainProgress: progress,
progressText: `${t('Search in file')} ${completedSteps} / ${totalSteps}`,
secondaryProgress: 0
});
}

const results = await searchTextByBatch({
query,
queryEmbedding,
file,
progressCallback: secProgressCallback,
threadId
});

if (results && results.length > 0) {
const bestMatch = results[0];
bestMatches.push({
...file,
firstCharIndex: bestMatch.firstCharIndex,
score: bestMatch.score,
text: getFirstChunk(file.content.slice(
bestMatch.firstCharIndex,
bestMatch.firstCharIndex + MAX_CHUNK_LENGTH
)),
otherSearchResults: results
});
}
} catch (error) {
console.error(`Error searching file ${file.fileId}:`, error);
}
}
};

// Process small files with semantic search
await processFiles(smallFileFiles);
if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);

// Process big files with semantic search
await processFiles(bigFileFiles);
if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);

// Process Google Drive files
if (gDriveFiles.length > 0) {
try {
// Check internet connectivity
const state = await fetch(); // Ensure fetch is correctly implemented
if (!state.isConnected) {
Toast.show({
type: 'infoToast',
text1: t('Google Drive files are missing from search results because there is no internet connection'),
visibilityTime: 5000,
topOffset: 30,
});
} else {
// Fetch file details including size
const gDriveFileDetails = await gdriveFileManager.getFilesDitails(gDriveFiles.map(file => file.fileId));

// Split gDriveFiles into small and big based on size
const smallGDriveFiles = [];
const bigGDriveFiles = [];

for (const file of gDriveFiles) {
const details = gDriveFileDetails[file.fileId];
if (!details) {
continue;
}

if (details.size < SMALL_FILE_SIZE) { // Adjust the threshold as needed
smallGDriveFiles.push(file);
} else {
bigGDriveFiles.push(file);
}
}


// Process small Google Drive files first
await processFiles(smallGDriveFiles, gDriveFileDetails);
if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

// Update search results after processing small GDrive files
bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);

// Process big Google Drive files
await processFiles(bigGDriveFiles, gDriveFileDetails);
if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

// Update search results after processing big GDrive files
bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);
}
} catch (error) {
console.error('Error fetching internet state or processing GDrive files:', error);
}
}

if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

setProgresses({
mainProgress: 100,
progressText: t('Done'),
secondaryProgress: 0
});

bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);

} catch (error) {
throw new Error(error.message)
}
}

function getFirstChunk(text) {
const length = text.length;

const minEnd = MIN_CHUNK_LENGTH;
const maxEnd = Math.min(MAX_CHUNK_LENGTH, length);

let splitPosition = -1;

// Variables to store positions of preferred break points
let newlinePos = -1;
let sentenceEndPos = -1;
let spacePos = -1;

// Loop from maxEnd down to minEnd to find the best split position
for (let i = maxEnd - 1; i >= minEnd; i--) {
const c = text[i];

if (newlinePos === -1 && c === '\n') {
newlinePos = i + 1;
// Found the highest priority split, can break early
break;
}

if (sentenceEndPos === -1 && (c === '.' || c === '!' || c === '?')) {
const nextChar = text[i + 1];
if (nextChar === ' ' || nextChar === '\n' || nextChar === undefined) {
sentenceEndPos = i + 1;
}
}

if (spacePos === -1 && c === ' ') {
spacePos = i + 1;
}
}

// Determine the split position based on the highest priority available
if (newlinePos !== -1) {
splitPosition = newlinePos;
} else if (sentenceEndPos !== -1) {
splitPosition = sentenceEndPos;
} else if (spacePos !== -1) {
splitPosition = spacePos;
} else {
splitPosition = maxEnd;
}

// Extract the first chunk
const chunk = text.slice(0, splitPosition);

return chunk;
}



I also have this function that does semantic searches on file names:

async function aiSearchLite(query, hasPermission) {

const threadId = (currentThreadId.current = performance.now());

if (!query) {
showAllFiles(hasPermission);
return;
}

const queryVector = await getFileNameEmbedding(query);
if (queryVector === null) {
Alert.alert(
t('Error'),
t('For some reason, an error occurred while performing the search. It may help to restart the application.'),
[],
{ cancelable: true }
);
return;
}

currentSearchResultType.current = SearchResultTypes.AI_LITE;

try {

const rows = await executeQuery(
"SELECT fileId, isAiSearchable, fileNameEmbedding, relativeFilePath, fileType, email FROM files"
);

setIsSearching(true);
setSearchList([]);
miniProgress.current.setProgress(null);
const length = rows.length - 1;
const gDrivePathes = [];

let startTimeForProgress = performance.now();
let isProgressShown = false;
const setProgress = (i, totalSteps) => {
return new Promise((resolve) => {
miniProgress.current.setProgress((i / totalSteps) * 100);
setTimeout(resolve, 0);
});
};


for (let i = 0; i <= length; i++) {
const row = rows.item(i);
const fileId = row.fileId;
const fileType = row.fileType;
const fileNameEmbedding = row.fileNameEmbedding;
let content = null;
let isError = false;
let embedding = null;

embedding = fileNameEmbedding.split(",").map(Number);

if (embedding.length !== EMBEDDING_LENGTH) {
const name = getFileName(fileType, row.relativeFilePath);
embedding = await getFileNameEmbedding(name);
const stringEmbedding = embedding.toString();

updateFileNameEmbedding(fileId, stringEmbedding).catch((error) => {
console.error(error);
});
}

let score = cosineSimilarity(embedding, queryVector);

if (fileType !== FileType.G_DRIVE_FILE) {
if (hasPermission || fileType !== FileType.SHARED_FILE) {
try {
if (await localFileManager.fileExists(fileId)) {
content = await localFileManager.getFileStartContent(fileId, subTextLength);
} else {
deleteFromBase(fileId);
}
} catch (error) {
console.error(error);
content = t('For some reason this file could not be read.');
isError = true;
}
} else {
content = t('This content is not available because you have not granted permission to access this file.');
isError = true;
}
} else {
if (userRef.current === null || userRef.current.email !== row.email) {
content = t('You are signed out of the Google account where this file resides, so it is unavailable.');
isError = true;
} else {
content = t('Loading...');
gDrivePathes.push({ fileId, score, email: row.email });
}
}

if (content !== null) {
const item = {
...row,
text: content,
queryIndex: { start: 0, end: 0 },
score,
isError,
};

if (currentThreadId.current !== threadId) {
return;
}

setSearchList((searchList) => {
const index = searchList.findIndex((listItem) => item.score > listItem.score);
if (index !== -1) {
searchList.splice(index, 0, item);
} else {
searchList.push(item);
}
return searchList;
});
}


if(performance.now()-startTimeForProgress > 1000) {
isProgressShown = true;
startTimeForProgress = performance.now();
await setProgress(i, length);
}

}

if(isProgressShown) {
await setProgress(1, 1);
setTimeout(()=>miniProgress.current.setProgress(null), 1000);
}

setIsSearching(false);
setRerender((r) => !r);

gDrivePathes.sort((a, b) => b.score - a.score);

if (currentThreadId.current !== threadId) return;
await uploadContentToListFromGDrive(gDrivePathes, threadId);
} catch (error) {
console.error(error);
Alert.alert(
t('Error'),
t('Something went wrong, failed to display search results.'),
[],
{ cancelable: true }
);
}
}



I show the results of their performance in this way:

const showList = ()=>{
return(
<View style={mainStyles.viewForScrollView}>

<View style={mainStyles.viewViewForScrollView}>
<FlatList
inverted
horizontal={false}
contentContainerStyle={multiStyles.flatListContent}
style={multiStyles.flatList}
keyExtractor={keyExtr}
data={searchList}
renderItem={renderItem}
ref={flatListRef}
/>
</View>
{isLoading &&
<View style={{width:'100%', height:'100%', position: 'absolute', justifyContent: "center",}}>
<ActivityIndicator size={adjustSize(100)} color="#888" />
</View>}

</View>
);
}

const renderItem = ({ item, index }) => {
return(
<SearchedItem
index = {index}
item = {item}
onLongPress={() => selectFile(index)}
onPress={() => handlePress(item, index)}

headStyle = {[item.isError && multiStyles.errorItemHead, item.isSelected && multiStyles.selectItemHead]}
bodyStyle = {[item.isError && multiStyles.errorItemBody, item.isSelected && multiStyles.selectItemBody]}
bottomStyle = {[item.isError && multiStyles.errorItemBottom, item.isSelected && multiStyles.selectItemBottom]}
/>
);
};


//SearchedItem.js

import React, { Component } from 'react';
import { ScrollView, Text, View, TouchableOpacity, ActivityIndicator, StyleSheet } from 'react-native';

import { EditorIcons } from './Icons';

import multiStyles from '../../styles/multifileSearchStyle';
import mainStyles from '../../styles/mainStyles';
import { isProApp } from '../appSwitcher';

import { HOME_FOLDER_NAME, FOLDER_NAME_SEPARATOR,
GOOGLE_DRIVE, SHARED_FOLDER_NAME,
IMPORTED_FILES_LABEL} from '../constantsAndFunctions';

import { FileType } from '../AppFile';

import { adjustSize } from '../../styles/adjustFontSize';
import { t } from '../../language/i18n';
import { getFirstChunk } from '../AISearchByFileContent';
import { MAX_CHUNK_LENGTH } from '../constantsAndFunctions';


export class SearchedItem extends Component {
shouldComponentUpdate(nextProps, nextState){

const item = this.props.item;
const nextItem = nextProps.item;

const queryIndex = item.queryIndex === undefined ? {start:0, end:0} : item.queryIndex;
const nextQueryIndex = nextItem.queryIndex === undefined ? {start:0, end:0} : nextItem.queryIndex;


if(item.fileId !== nextItem.fileId) return true;

if(item.text!==nextItem.text) return true;

if(queryIndex.end!==nextQueryIndex.end || queryIndex.start!==nextQueryIndex.start) return true;

if(item.isAiSearchable!==nextItem.isAiSearchable) return true;

if(item.isLoading!==nextItem.isLoading) return true;

if(item.isOpening!==nextItem.isOpening) return true;

if(item.pathQuery!==nextItem.pathQuery) return true;

if(item.isSelected!==nextItem.isSelected) return true;

if(item.isError!==nextItem.isError) return true;

if(this.props.index!==nextProps.index) return true;

// Keep existing checks
if (this.state.currentMatchIndex !== nextState.currentMatchIndex) return true;
if (this.state.score !== nextState.score) return true;
if (this.state.chunk !== nextState.chunk) return true;

return false;

}

constructor(props){

super(props);
this.scrollV = React.createRef();



// Keep track of which match in otherSearchResults we are showing
this.state = {
score: props.item.score,
currentMatchIndex: 0,
chunk: undefined,
};

this.handlePrevMatch = this.handlePrevMatch.bind(this);
this.handleNextMatch = this.handleNextMatch.bind(this);

}

/**
* If we click on the left arrow, show the previous result.
*/
handlePrevMatch() {

}

/**
* If we click on the right arrow, show the next result.
*/
handleNextMatch() {

}

/**
* Helper function to show "Ai" if isAiSearchable===1 in Pro version.
*/
showAiSearchable = () => {
if (!isProApp) return null;
if (this.props.item.isAiSearchable === 1) {
return (
<Text style={{ paddingHorizontal:adjustSize(5), fontSize:adjustSize(14), color:'black' }}>
Ai
</Text>
);
}
return null;
}

render() {
const item = this.props.item;
let fileIdToShow = item.fileId;
const { score, chunk, currentMatchIndex } = this.state;

const displayChunk = chunk ? chunk : this.props.item.text;

const queryIndex = item.queryIndex === undefined ? {start:0, end:0} : item.queryIndex;

if(item.fileType===FileType.HOME_FILE) {
fileIdToShow = t(HOME_FOLDER_NAME)+'/'+item.relativeFilePath;
} else if (item.fileType===FileType.G_DRIVE_FILE) {
fileIdToShow = t(GOOGLE_DRIVE) + '/' + item.relativeFilePath.replace(new RegExp(FOLDER_NAME_SEPARATOR, "g"), "/");
} else if(item.fileType===FileType.SHARED_FILE) {
fileIdToShow = t(SHARED_FOLDER_NAME)+'/'+item.relativeFilePath;
} else if(item.fileType===FileType.NEW_FILE) {
fileIdToShow = '* '+item.relativeFilePath;
} else if(item.fileType===FileType.IMPORTED_FILE) {
fileIdToShow = '* '+ t(IMPORTED_FILES_LABEL)+'/'+item.relativeFilePath;
}


//Used to highlight in red for character-by-character searches
const pathQuery = item.pathQuery;
if(pathQuery !== undefined) {
var queryMatch = fileIdToShow.toLowerCase().lastIndexOf(pathQuery.toLowerCase());
}

return (
<View style={multiStyles.searchedItem}>
<View style={[multiStyles.searchedItemBorder, item.isError && {borderColor:'#FF737D'}, item.isSelected && {borderColor:'#99D1FF'}]}>
<View
style={[multiStyles.normalItemHead, this.props.headStyle, item.isLoading && {paddingRight:0}]}//this.props.style.borderColor '#99D1FF' {backgroundColor:this.props.style.borderColor}
>
<ScrollView
horizontal={true}
showsHorizontalScrollIndicator={false}
ref={this.scrollV}
//style={{backgroundColor:'blue'}}
onContentSizeChange={() => {this.scrollV.current?.scrollToEnd()}}
>
<View>
{ queryMatch!==undefined ?

<Text style={mainStyles.showPathText}>
<Text>{fileIdToShow.substring(0, queryMatch)}</Text>
<Text style={mainStyles.match}>{fileIdToShow.substring(queryMatch, queryMatch + pathQuery.length)}</Text>
<Text>{fileIdToShow.substring(queryMatch + pathQuery.length, fileIdToShow.length)}</Text>
</Text>
:
<Text style={mainStyles.showPathText}>{fileIdToShow} </Text>
}

</View>
</ScrollView>

{item.isLoading &&
<View style={{
paddingRight:adjustSize(7),
flexDirection:'row',
justifyContent:'center',
alignItems:'center',
}}>
<ActivityIndicator size={adjustSize(14)} color="#000" />
</View>}
{item.isLoading && this.scrollV.current?.scrollToEnd()}
{this.showAiSearchable()}
</View>
<TouchableOpacity onLongPress={this.props.onLongPress}
delayLongPress={1000} onPress={this.props.onPress}>
<View style={[multiStyles.normalItemBodyView, !item.email && {borderBottomLeftRadius:adjustSize(20)}, this.props.bodyStyle]}>
<Text style={multiStyles.normalItemBodyText}>
<Text>{displayChunk.substring(0, queryIndex.start)}</Text>
<Text style={mainStyles.match}>{displayChunk.substring(queryIndex.start, queryIndex.end)}</Text>

<Text>
{displayChunk.substring(queryIndex.end, displayChunk.length)}
</Text>
</Text>
</View>
{item.email &&
<View style={[multiStyles.normalItemBottom, this.props.bottomStyle]}>
<Text style={multiStyles.normalItemBottomText}>{item.email}</Text>
</View>}
{item.isOpening &&
<View style={{width:'100%', height:'100%', position: 'absolute', justifyContent: "center", backgroundColor:'rgba(255, 255, 255, 0.5)'}}>
<ActivityIndicator size={adjustSize(32)} color="#888" />
</View>}
</TouchableOpacity>
</View>

{
item.score !==undefined &&
<View style={styles.scoreContainer}>
<View style={styles.textContainer}>
<Text style={styles.text}>{t("Match: ") + (score*100).toFixed(2) + '%'}</Text>
</View>

{
item.otherSearchResults && item.otherSearchResults.length > 1 &&
(<View style={styles.navigationContainer}>
<TouchableOpacity
style={[styles.button, styles.button1]}
onPress={this.handlePrevMatch}>
<EditorIcons name="left_arrow_0_padding" size={22} color="#333"/>
</TouchableOpacity>
<View style={styles.textContainer}><Text style={styles.text}>{currentMatchIndex+1}/{item.otherSearchResults.length}</Text></View>
<TouchableOpacity
style={[styles.button, styles.button2]}
onPress={this.handleNextMatch}
>
<EditorIcons name="right_arrow_0_padding" size={22} color="#333"/>
</TouchableOpacity>
</View>)
}
</View>
}

</View>
);
}
}

const styles = StyleSheet.create({
scoreContainer: { flexDirection:'row', paddingTop:3, alignItems:'start', justifyContent:'space-between', width:'100%' },
textContainer: {
height:adjustSize(22), flexDirection:'row', alignItems:'center' },
text: {
color:'#000' },
navigationContainer: { flexDirection:'row', alignItems:'flex-start', justifyContent:'center' },
button: {
height: adjustSize(48),
width: adjustSize(48),
minHeight:48,
minWidth:48, flexDirection:'row', alignItems:'flex-start' },
button1: { justifyContent:'flex-start', },
button2: { justifyContent:'flex-end', },
})

I need to finalize and where I need to correct the SearchedItem component. Make it so that when you click on the left_arrow_0_padding and right_arrow_0_padding arrows, the information in SearchedItem will be updated showing the new text chunk, its score and its sequence number in the item.otherSearchResults array.
I also want you to take into account the fact that I don't want to pass the content of the whole file to the SearchedItem component, in order to save memory. So I want the content of the whole file to be loaded only when the user clicks on one of the arrows.