Fetch chunk for matches

I have this module for semantic search through the contents of text files:

// AISearchByFileContent.js

import { t } from "../language/i18n";
import threadManagerInstance from "./ThreadManager";
import { searchTextByBatch } from "./SemanticSearch";
import { FileType } from './AppFile';
import { fetch } from "@react-native-community/netinfo";
import { deleteFromBase, executeQuery } from './DBManager';
import { getGDriveFileContentAndModifiedTimeOrDeleteFromBase, MAX_CHUNK_LENGTH, MIN_CHUNK_LENGTH } from './constantsAndFunctions';
import { SearchResultTypes } from '../screens/multifileSearch';
import ProgressBar from './GUI_Elements/ProgressBar';
import TextEmbenderInstance from './AI_Modules/TextEmbender';
import { gdriveFileManager, localFileManager } from './file_manager/FileManagerFactory';
import Toast from 'react-native-toast-message';

const setProgresses = ({ mainProgress, progressText, secondaryProgress, closeHandler }) => {

mainProgress !== undefined && ProgressBar.setProgress(mainProgress);
progressText !== undefined && ProgressBar.setText(progressText);
secondaryProgress !== undefined && ProgressBar.setSecondaryProgress(secondaryProgress);
closeHandler !== undefined && ProgressBar.setCloseHandler(closeHandler);

}

// Helper function to get file content based on file type
async function getFileContent(file, gDriveFileDetails = null) {
if (file.fileType === FileType.G_DRIVE_FILE) {
const {content, modificationTime} = await getGDriveFileContentAndModifiedTimeOrDeleteFromBase(
file.fileId,
gDriveFileDetails[file.fileId]?.modifiedTime
);

text
file.modificationTime = modificationTime;
return content;

} else {
file.content = await localFileManager.getFileContent(file.fileId);
return file.content;
}
}

export async function AISearchByFileContent(
query, hasPermission, setSearchList, setRerender,
currentSearchResultType, userRef, threadId
) {

const SMALL_FILE_SIZE = 2300;

// Set the current search result type to semantic
// Assuming you have a state or context to manage this
currentSearchResultType.current = SearchResultTypes.AI;

setSearchList([]);

let startTime = performance.now();

setProgresses({
mainProgress: 0,
progressText: t('Analyzing available files'),
closeHandler: ()=>{
threadManagerInstance.clearCurrentThreadId();
ProgressBar.setText(t('Stopping...'));
}
});

try {
// Fetch all files from the database
const allFilesRows = await executeQuery(
'SELECT fileId, isAiSearchable, fileType, lastOpening, relativeFilePath, email, modifiedTimeBeforeIndexing FROM files ORDER BY lastOpening DESC'
);

text
const gDriveFiles = [];
const bigFileFiles = [];
const smallFileFiles = [];

// Categorize files
for (let i = 0; i < allFilesRows.length; i++) {
  const item = allFilesRows.item(i);
  const fileId = item.fileId;

  if (item.fileType !== FileType.G_DRIVE_FILE) {
    // If we don't have permission and file is SHARED_FILE, skip reading
    if (!hasPermission && item.fileType === FileType.SHARED_FILE) {
      continue;
    }

    try {
      if (await localFileManager.fileExists(fileId)) {

        const fileDitails = await localFileManager.getFileDetails(fileId);
        item.modificationTime = fileDitails.modifiedTime;

        if (fileDitails.size < SMALL_FILE_SIZE) { 
          smallFileFiles.push(item);
        } else {
          bigFileFiles.push(item);
        }
      } else {
        await deleteFromBase(fileId); 
      }
    } catch (error) {
      console.error(`Error accessing file ${fileId}:`, error);
      continue;
    }
  } else {
    if (userRef.current !== null && userRef.current.email === item.email) {
      gDriveFiles.push(item);
    }
  }
}

// Calculate total steps for progress
let totalSteps = smallFileFiles.length + bigFileFiles.length + gDriveFiles.length;


if( totalSteps === 0 ) return;


const queryEmbedding = await TextEmbenderInstance.getSentenceEmbedding(query);

let completedSteps = 0;

// Function to update progress
const updateProgress = () => {

  if(performance.now()-startTime > 1000) {
    startTime = performance.now();
    const progress = ((completedSteps-1) / totalSteps) * 100;
    setProgresses({ 
      mainProgress: progress, 
      progressText: `${t('Search in file')} ${completedSteps} / ${totalSteps}`,
      secondaryProgress: 0 
    });
  }
};

const secondaryProgressCallback = (secondaryProgress, progressText) => {
  if(performance.now()-startTime > 1000) {
    startTime = performance.now();
    setProgresses({ progressText: `${completedSteps} / ${totalSteps}  |  ${progressText}`, secondaryProgress });
  }
}

// Collect best matches from each file
const bestMatches = [];

// Function to process a list of files
const processFiles = async (files, gDriveFileDetails = null) => {
  for (let i = 0; i < files.length; i++) {
    if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

    completedSteps += 1;
    updateProgress();

    const file = files[i];
    
    try {
      file.content = await getFileContent(file, gDriveFileDetails);
      let secProgressCallback = undefined;
      if(file.modificationTime !== file.modifiedTimeBeforeIndexing) {
        secProgressCallback = secondaryProgressCallback;

        const progress = (completedSteps / totalSteps) * 100;
        setProgresses({ 
          mainProgress: progress, 
          progressText: `${t('Search in file')} ${completedSteps} / ${totalSteps}`,
          secondaryProgress: 0 
        });
      }

      const results = await searchTextByBatch({
        query, 
        queryEmbedding, 
        file, 
        progressCallback: secProgressCallback, 
        threadId
      });
      
      if (results && results.length > 0) {
        const bestMatch = results[0];
        bestMatches.push({
          ...file,
          firstCharIndex: bestMatch.firstCharIndex,
          score: bestMatch.score,
          text: getFirstChunk(file.content.slice(
            bestMatch.firstCharIndex, 
            bestMatch.firstCharIndex + MAX_CHUNK_LENGTH
          )),
          otherSearchResults: results
        });
      }
    } catch (error) {
      console.error(`Error searching file ${file.fileId}:`, error);
    }
  }
};

// Process small files with semantic search
await processFiles(smallFileFiles);
if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);

// Process big files with semantic search
await processFiles(bigFileFiles);
if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);

// Process Google Drive files
if (gDriveFiles.length > 0) {
  try {
    // Check internet connectivity
    const state = await fetch(); // Ensure fetch is correctly implemented
    if (!state.isConnected) {
        Toast.show({ 
          type: 'infoToast',
          text1: t('Google Drive files are missing from search results because there is no internet connection'),
          visibilityTime: 5000,
          topOffset: 30,
        });
    } else {
      // Fetch file details including size
      const gDriveFileDetails = await gdriveFileManager.getFilesDitails(gDriveFiles.map(file => file.fileId));

      // Split gDriveFiles into small and big based on size
      const smallGDriveFiles = [];
      const bigGDriveFiles = [];

      for (const file of gDriveFiles) {
        const details = gDriveFileDetails[file.fileId];
        if (!details) {
          continue;
        }

        if (details.size < SMALL_FILE_SIZE) { // Adjust the threshold as needed
          smallGDriveFiles.push(file);
        } else {
          bigGDriveFiles.push(file);
        }
      }


      // Process small Google Drive files first
      await processFiles(smallGDriveFiles, gDriveFileDetails);
      if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

      // Update search results after processing small GDrive files
      bestMatches.sort((a, b) => b.score - a.score);
      setSearchList([...bestMatches]);
      setRerender(r=>!r);

      // Process big Google Drive files
      await processFiles(bigGDriveFiles, gDriveFileDetails);
      if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

      // Update search results after processing big GDrive files
      bestMatches.sort((a, b) => b.score - a.score);
      setSearchList([...bestMatches]);
      setRerender(r=>!r);
    }
  } catch (error) {
    console.error('Error fetching internet state or processing GDrive files:', error);
  }
}

if (threadManagerInstance.isThreadNotCurrent(threadId)) return;

setProgresses({ 
  mainProgress: 100, 
  progressText: t('Done'),
  secondaryProgress: 0 
});

bestMatches.sort((a, b) => b.score - a.score);
setSearchList([...bestMatches]);
setRerender(r=>!r);

} catch (error) {
throw new Error(error.message)
}
}

function getFirstChunk(text) {
const length = text.length;

const minEnd = MIN_CHUNK_LENGTH;
const maxEnd = Math.min(MAX_CHUNK_LENGTH, length);

let splitPosition = -1;

// Variables to store positions of preferred break points
let newlinePos = -1;
let sentenceEndPos = -1;
let spacePos = -1;

// Loop from maxEnd down to minEnd to find the best split position
for (let i = maxEnd - 1; i >= minEnd; i--) {
const c = text[i];

text
if (newlinePos === -1 && c === '\n') {
  newlinePos = i + 1;
  // Found the highest priority split, can break early
  break;
}

if (sentenceEndPos === -1 && (c === '.' || c === '!' || c === '?')) {
  const nextChar = text[i + 1];
  if (nextChar === ' ' || nextChar === '\n' || nextChar === undefined) {
    sentenceEndPos = i + 1;
  }
}

if (spacePos === -1 && c === ' ') {
  spacePos = i + 1;
}

}

// Determine the split position based on the highest priority available
if (newlinePos !== -1) {
splitPosition = newlinePos;
} else if (sentenceEndPos !== -1) {
splitPosition = sentenceEndPos;
} else if (spacePos !== -1) {
splitPosition = spacePos;
} else {
splitPosition = maxEnd;
}

// Extract the first chunk
const chunk = text.slice(0, splitPosition);

return chunk;
}

I also have this function that does semantic searches on file names:

async function aiSearchLite(query, hasPermission) {

text
	const threadId = (currentThreadId.current = performance.now());
  
	if (!query) {
	  showAllFiles(hasPermission);
	  return;
	}
  
	const queryVector = await getFileNameEmbedding(query);
	if (queryVector === null) {
	  Alert.alert(
		t('Error'),
		t('For some reason, an error occurred while performing the search. It may help to restart the application.'),
		[],
		{ cancelable: true }
	  );
	  return;
	}
  
	currentSearchResultType.current = SearchResultTypes.AI_LITE;
	
	try {

		const rows = await executeQuery(
			"SELECT fileId, isAiSearchable, fileNameEmbedding, relativeFilePath, fileType, email FROM files"
		);

		setIsSearching(true);
		setSearchList([]);
		miniProgress.current.setProgress(null);
		const length = rows.length - 1;
		const gDrivePathes = [];

		let startTimeForProgress = performance.now();
		let isProgressShown = false;
		const  setProgress = (i, totalSteps) => {
			return new Promise((resolve) => {
				miniProgress.current.setProgress((i / totalSteps) * 100);
				setTimeout(resolve, 0);
			});
		};
		
	
		for (let i = 0; i <= length; i++) {
			const row = rows.item(i);
			const fileId = row.fileId;
			const fileType = row.fileType;
			const fileNameEmbedding = row.fileNameEmbedding;
			let content = null;
			let isError = false;
			let embedding = null;
	
			embedding = fileNameEmbedding.split(",").map(Number);
	
			if (embedding.length !== EMBEDDING_LENGTH) {
			const name = getFileName(fileType, row.relativeFilePath);
			embedding = await getFileNameEmbedding(name);
			const stringEmbedding = embedding.toString();
	
			updateFileNameEmbedding(fileId, stringEmbedding).catch((error) => {
				console.error(error);
			});
			}
	
			let score = cosineSimilarity(embedding, queryVector);
	
			if (fileType !== FileType.G_DRIVE_FILE) {
				if (hasPermission || fileType !== FileType.SHARED_FILE) {
					try {
						if (await localFileManager.fileExists(fileId)) {
							content = await localFileManager.getFileStartContent(fileId, subTextLength);
						} else {
							deleteFromBase(fileId);
						}
					} catch (error) {
						console.error(error);
						content = t('For some reason this file could not be read.');
						isError = true;
					}
				} else {
					content = t('This content is not available because you have not granted permission to access this file.');
					isError = true;
				}
			} else {
				if (userRef.current === null || userRef.current.email !== row.email) {
					content = t('You are signed out of the Google account where this file resides, so it is unavailable.');
					isError = true;
				} else {
					content = t('Loading...');
					gDrivePathes.push({ fileId, score, email: row.email });
				}
			}
	
			if (content !== null) {
				const item = {
					...row,
					text: content,
					queryIndex: { start: 0, end: 0 },
					score,
					isError,
				};
		
				if (currentThreadId.current !== threadId) {
					return;
				}
		
				setSearchList((searchList) => {
					const index = searchList.findIndex((listItem) => item.score > listItem.score);
					if (index !== -1) {
						searchList.splice(index, 0, item);
					} else {
						searchList.push(item);
					}
					return searchList;
				});
			}

			
			if(performance.now()-startTimeForProgress > 1000) {
				isProgressShown = true;
				startTimeForProgress = performance.now();
				await setProgress(i, length);
			}
			
		}

		if(isProgressShown) {
			await setProgress(1, 1);
			setTimeout(()=>miniProgress.current.setProgress(null), 1000);
		}

		setIsSearching(false);
		setRerender((r) => !r);
	
		gDrivePathes.sort((a, b) => b.score - a.score);
	
		if (currentThreadId.current !== threadId) return;
		await uploadContentToListFromGDrive(gDrivePathes, threadId);
	} catch (error) {
	  console.error(error);
	  Alert.alert(
		t('Error'),
		t('Something went wrong, failed to display search results.'),
		[],
		{ cancelable: true }
	  );
	}
}

I show the results of their performance in this way:

text
const showList = ()=>{
    return(
        <View style={mainStyles.viewForScrollView}>

          <View style={mainStyles.viewViewForScrollView}>
            <FlatList 
				inverted
				horizontal={false}
				contentContainerStyle={multiStyles.flatListContent}
				style={multiStyles.flatList} 
				keyExtractor={keyExtr}
				data={searchList}
				renderItem={renderItem}
				ref={flatListRef} 
            />
          </View>
		  {isLoading && 
				<View style={{width:'100%', height:'100%',  position: 'absolute', justifyContent: "center",}}>
					<ActivityIndicator size={adjustSize(100)} color="#888" />
				</View>}
          
        </View>
    );
}

const renderItem = ({ item, index }) => { 
    return(
        <SearchedItem
			index = {index}
			item = {item}
			onLongPress={() => selectFile(index)}
			onPress={() => handlePress(item, index)}

			headStyle = {[item.isError && multiStyles.errorItemHead, item.isSelected && multiStyles.selectItemHead]}
			bodyStyle = {[item.isError && multiStyles.errorItemBody, item.isSelected && multiStyles.selectItemBody]}
			bottomStyle = {[item.isError && multiStyles.errorItemBottom, item.isSelected && multiStyles.selectItemBottom]}
		/>
    );
};

//SearchedItem.js

import React, { Component } from 'react';
import { ScrollView, Text, View, TouchableOpacity, ActivityIndicator, StyleSheet } from 'react-native';

import { EditorIcons } from './Icons';

import multiStyles from '../../styles/multifileSearchStyle';
import mainStyles from '../../styles/mainStyles';
import { isProApp } from '../appSwitcher';

import { HOME_FOLDER_NAME, FOLDER_NAME_SEPARATOR,
GOOGLE_DRIVE, SHARED_FOLDER_NAME,
IMPORTED_FILES_LABEL} from '../constantsAndFunctions';

import { FileType } from '../AppFile';

import { adjustSize } from '../../styles/adjustFontSize';
import { t } from '../../language/i18n';
import { getFirstChunk } from '../AISearchByFileContent';
import { MAX_CHUNK_LENGTH } from '../constantsAndFunctions';

export class SearchedItem extends Component {
shouldComponentUpdate(nextProps, nextState){

text
  const item = this.props.item;
  const nextItem = nextProps.item;

  const queryIndex = item.queryIndex === undefined ? {start:0, end:0} : item.queryIndex;
  const nextQueryIndex = nextItem.queryIndex === undefined ? {start:0, end:0} : nextItem.queryIndex;
  

  if(item.fileId !== nextItem.fileId) return true;

  if(item.text!==nextItem.text) return true;

  if(queryIndex.end!==nextQueryIndex.end || queryIndex.start!==nextQueryIndex.start) return true;

  if(item.isAiSearchable!==nextItem.isAiSearchable) return true;

  if(item.isLoading!==nextItem.isLoading) return true;

  if(item.isOpening!==nextItem.isOpening) return true;

  if(item.pathQuery!==nextItem.pathQuery) return true;

  if(item.isSelected!==nextItem.isSelected) return true;

  if(item.isError!==nextItem.isError) return true;

  if(this.props.index!==nextProps.index) return true;

  // Keep existing checks
  if (this.state.currentMatchIndex !== nextState.currentMatchIndex) return true;
  if (this.state.score !== nextState.score) return true;
  if (this.state.chunk !== nextState.chunk) return true;

  return false;
  
}

constructor(props){

    super(props);
    this.scrollV = React.createRef();



    // Keep track of which match in otherSearchResults we are showing
    this.state = {
      score: props.item.score,
      currentMatchIndex: 0,
      chunk: undefined,
    };

    this.handlePrevMatch = this.handlePrevMatch.bind(this);
    this.handleNextMatch = this.handleNextMatch.bind(this);

}

/**
 * If we click on the left arrow, show the previous result.
 */
handlePrevMatch() {

}

/**
 * If we click on the right arrow, show the next result.
 */
handleNextMatch() {

}

/**
 * Helper function to show "Ai" if isAiSearchable===1 in Pro version.
 */
showAiSearchable = () => {
  if (!isProApp) return null;
  if (this.props.item.isAiSearchable === 1) {
    return (
      <Text style={{ paddingHorizontal:adjustSize(5), fontSize:adjustSize(14), color:'black' }}>
        Ai
      </Text>
    );
  } 
  return null;
}
     
render() {
  const item = this.props.item;
  let fileIdToShow = item.fileId;
  const { score, chunk, currentMatchIndex } = this.state;

  const displayChunk = chunk ? chunk : this.props.item.text;

  const queryIndex = item.queryIndex === undefined ? {start:0, end:0} : item.queryIndex;
  
  if(item.fileType===FileType.HOME_FILE) {
    fileIdToShow = t(HOME_FOLDER_NAME)+'/'+item.relativeFilePath;
  } else  if (item.fileType===FileType.G_DRIVE_FILE) {
    fileIdToShow = t(GOOGLE_DRIVE) + '/' + item.relativeFilePath.replace(new RegExp(FOLDER_NAME_SEPARATOR, "g"), "/");
  } else if(item.fileType===FileType.SHARED_FILE) {
    fileIdToShow = t(SHARED_FOLDER_NAME)+'/'+item.relativeFilePath;
  } else if(item.fileType===FileType.NEW_FILE) {
    fileIdToShow = '* '+item.relativeFilePath;
  } else if(item.fileType===FileType.IMPORTED_FILE) {
    fileIdToShow = '* '+ t(IMPORTED_FILES_LABEL)+'/'+item.relativeFilePath;
  }


  //Used to highlight in red for character-by-character searches
  const pathQuery = item.pathQuery;
  if(pathQuery !== undefined) {
    var queryMatch = fileIdToShow.toLowerCase().lastIndexOf(pathQuery.toLowerCase());
  }

  return (
      <View style={multiStyles.searchedItem}>
          <View style={[multiStyles.searchedItemBorder, item.isError && {borderColor:'#FF737D'}, item.isSelected && {borderColor:'#99D1FF'}]}>
            <View
              style={[multiStyles.normalItemHead, this.props.headStyle, item.isLoading && {paddingRight:0}]}//this.props.style.borderColor '#99D1FF' {backgroundColor:this.props.style.borderColor}
              >
              <ScrollView
                  horizontal={true}
                  showsHorizontalScrollIndicator={false}
                  ref={this.scrollV}
                  //style={{backgroundColor:'blue'}}
                  onContentSizeChange={() => {this.scrollV.current?.scrollToEnd()}}
                  >
                  <View>
                    { queryMatch!==undefined ?
                      
                      <Text style={mainStyles.showPathText}>
                        <Text>{fileIdToShow.substring(0, queryMatch)}</Text>
                        <Text style={mainStyles.match}>{fileIdToShow.substring(queryMatch, queryMatch + pathQuery.length)}</Text>
                        <Text>{fileIdToShow.substring(queryMatch + pathQuery.length, fileIdToShow.length)}</Text>
                      </Text>
                      :
                      <Text style={mainStyles.showPathText}>{fileIdToShow} </Text>
                    }
                    
                  </View>
              </ScrollView>
              
              {item.isLoading && 
                  <View style={{
                          paddingRight:adjustSize(7),
                          flexDirection:'row',
                          justifyContent:'center',
                          alignItems:'center',
                           }}>
                          <ActivityIndicator size={adjustSize(14)} color="#000" />
                  </View>}
              {item.isLoading && this.scrollV.current?.scrollToEnd()}
              {this.showAiSearchable()}
            </View>
            <TouchableOpacity onLongPress={this.props.onLongPress}
                              delayLongPress={1000}  onPress={this.props.onPress}>
                <View style={[multiStyles.normalItemBodyView, !item.email && {borderBottomLeftRadius:adjustSize(20)}, this.props.bodyStyle]}>
                  <Text style={multiStyles.normalItemBodyText}>
                    <Text>{displayChunk.substring(0, queryIndex.start)}</Text>
                    <Text style={mainStyles.match}>{displayChunk.substring(queryIndex.start, queryIndex.end)}</Text>
      
                    <Text>
                      {displayChunk.substring(queryIndex.end, displayChunk.length)}
                    </Text>
                  </Text>
                </View>
                {item.email && 
                  <View style={[multiStyles.normalItemBottom, this.props.bottomStyle]}>
                      <Text style={multiStyles.normalItemBottomText}>{item.email}</Text>
                  </View>}
                {item.isOpening && 
                  <View style={{width:'100%', height:'100%',  position: 'absolute', justifyContent: "center", backgroundColor:'rgba(255, 255, 255, 0.5)'}}>
                          <ActivityIndicator size={adjustSize(32)} color="#888" />
                  </View>}
            </TouchableOpacity>
          </View>

          {
            item.score !==undefined &&
            <View style={styles.scoreContainer}>
              <View style={styles.textContainer}>
                <Text style={styles.text}>{t("Match: ") + (score*100).toFixed(2) + '%'}</Text>
              </View>

              {
                item.otherSearchResults && item.otherSearchResults.length > 1 && 
                (<View style={styles.navigationContainer}>
                  <TouchableOpacity 
                    style={[styles.button, styles.button1]}
                    onPress={this.handlePrevMatch}>
                    <EditorIcons name="left_arrow_0_padding" size={22} color="#333"/>
                  </TouchableOpacity>
                  <View style={styles.textContainer}><Text style={styles.text}>{currentMatchIndex+1}/{item.otherSearchResults.length}</Text></View>
                  <TouchableOpacity 
                    style={[styles.button, styles.button2]}
                    onPress={this.handleNextMatch}
                  >
                    <EditorIcons name="right_arrow_0_padding" size={22} color="#333"/>
                  </TouchableOpacity>
                </View>)
              }
            </View>
          }

      </View>
  );
}  

}

const styles = StyleSheet.create({
scoreContainer: { flexDirection:'row', paddingTop:3, alignItems:'start', justifyContent:'space-between', width:'100%' },
textContainer: {
height:adjustSize(22), flexDirection:'row', alignItems:'center' },
text: {
color:'#000' },
navigationContainer: { flexDirection:'row', alignItems:'flex-start', justifyContent:'center' },
button: {
height: adjustSize(48),
width: adjustSize(48),
minHeight:48,
minWidth:48, flexDirection:'row', alignItems:'flex-start' },
button1: { justifyContent:'flex-start', },
button2: { justifyContent:'flex-end', },
})

I need to finalize and where I need to correct the SearchedItem component. Make it so that when you click on the left_arrow_0_padding and right_arrow_0_padding arrows, the information in SearchedItem will be updated showing the new text chunk, its score and its sequence number in the item.otherSearchResults array.
I also want you to take into account the fact that I don't want to pass the content of the whole file to the SearchedItem component, in order to save memory. So I want the content of the whole file to be loaded only when the user clicks on one of the arrows.