/*==========================================================================*/
/* Copyright (c) 1999 - Sykes Enterprises Inc. - All rights reserved        */
/*--------------------------------------------------------------------------*/
/* Sykes grants you ("Licensee") a non-exclusive, royalty free, license to  */
/* use, modify and redistribute this software in source and binary code     */
/* form, provided that i) this copyright notice and license appear on all   */
/* copies of the software; and ii) Licensee does not utilize the software   */
/* in a manner which is disparaging to Sykes.                               */
/*                                                                          */
/* This software is provided "AS IS", without a warranty of any kind. ALL   */
/* EXPRESS OR IMPLIED CONDITIONS, REPRESENTATIONS AND WARRANTIES, INCLUDING */
/* ANY IMPLIED WARRANTY OF MERCHANTABILITY, FITNESS FOR A PARTICULAR        */
/* PURPOSE OR NON-INFRINGEMENT, ARE HEREBY EXCLUDED. SYKES AND ITS          */
/* LICENSORS SHALL NOT BE LIABLE FOR ANY DAMAGES SUFFERED BY LICENSEE AS A  */
/* RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE OR ITS           */
/* DERIVATIVES. IN NO EVENT WILL SYKES OR ITS LICENSORS BE LIABLE FOR ANY   */
/* LOST REVENUE, PROFIT OR DATA, OR FOR DIRECT, INDIRECT, SPECIAL,          */
/* CONSEQUENTIAL, INCIDENTAL OR PUNITIVE DAMAGES, HOWEVER CAUSED AND        */
/* REGARDLESS OF THE THEORY OF LIABILITY, ARISING OUT OF THE USE OF OR      */
/* INABILITY TO USE SOFTWARE, EVEN IF SYKES HAS BEEN ADVISED OF THE         */
/* POSSIBILITY OF SUCH DAMAGES.                                             */
/*==========================================================================*/

#include "okLiteTM.h"


/*--------------------------------------------------------------------------*/

const char* okLiteTM::ConvertToXML
(
	string&              p_rsText,
	const bool           p_bAttribute,
	const bool           p_bUnicodeCompatible
)
{
	string               sTmp;


	for ( int i=0, nLen=p_rsText.length(); i<nLen; i++ )
	{
		switch ( p_rsText[i] )
		{
			case '&':
				sTmp += "&amp;";
				continue;
			case '<':
				sTmp += "&lt;";
				continue;
			case '"':
				if ( p_bAttribute ) sTmp += "&quot;";
				else sTmp += p_rsText[i];
				continue;
			case '\'':
				if ( p_bAttribute ) sTmp += "&apos;";
				else sTmp += p_rsText[i];
				continue;
			default:
				if ( p_bUnicodeCompatible )
				{
					if (( p_rsText[i] > 0x7F ) && ( p_rsText[i] < 0xA0 ))
					{
						// Not iso-8859-1 characters
						sTmp += '?';
						continue;
					}
				}
				sTmp += p_rsText[i];
				continue;
		}
	}

	p_rsText = sTmp;
	return(p_rsText.c_str());
}


/*--------------------------------------------------------------------------*/

okLiteTM::okLiteTM ()
{
	m_pIndex = NULL;
	m_lIndexSize = 0;
	m_lCount = 0;
	m_pFile = NULL;
}


/*--------------------------------------------------------------------------*/

okLiteTM::~okLiteTM ()
{
	Close();
};


/*--------------------------------------------------------------------------*/

void okLiteTM::Free ()
{
	if ( m_pIndex != NULL ) free(m_pIndex);
	m_pIndex = NULL;
	m_lIndexSize = 0;
}


/*--------------------------------------------------------------------------*/

void okLiteTM::Close ()
{
	if ( m_pFile != NULL ) fclose(m_pFile);
	m_pFile = NULL;
	m_lCount = 0;
	Free();
}


/*--------------------------------------------------------------------------*/

void okLiteTM::SetInfo
(
	const char*          p_pszSrcLocale,
	const char*          p_pszTrgLocale,
	const char*          p_pszCodeset
)
{
	if ( p_pszSrcLocale != NULL ) m_sSrcLocale = p_pszSrcLocale;
	if ( p_pszTrgLocale != NULL ) m_sTrgLocale = p_pszTrgLocale;
	if ( p_pszCodeset != NULL ) m_sCodeset = p_pszCodeset;
};


/*--------------------------------------------------------------------------*/

bool okLiteTM::Open
(
	const char*          p_pszPath,
	const bool           p_bReadOnly,
	const char*          p_pszSrcLocale,
	const char*          p_pszTrgLocale,
	const char*          p_pszCodeset
)
{
#define LBUF 100

	char                 szBuf[LBUF+1];
	char                 szMode[4] = "rb";
	bool                 bExist = false;


	Close(); // Make sure we start fresh

	m_bReadOnly = p_bReadOnly;
	if ( !m_bReadOnly ) strcpy(szMode, "r+b");

	// Check if the file already exist
	if ( (m_pFile = fopen(p_pszPath, "rb")) != NULL )
	{
		bExist = true;
		fclose(m_pFile);
	}

	if ( !bExist && !p_bReadOnly )
	{
		// Need to create the file so we can open it in r+b mode
		if ( (m_pFile = fopen(p_pszPath, "wb")) == NULL )
		{
			// Cannot create the file
			return(false);
		}
		fclose(m_pFile);
	}

	if ( (m_pFile = fopen(p_pszPath, szMode)) == NULL )
	{
		return(false);
	}
	m_sPath = p_pszPath;

	// If the file does not exist: create it.
	if ( !bExist )
	{
		m_lIndexPos = okLTM_LHEADER;
		m_lIndexSize = 0;
		m_lCount = 0;
		SetInfo(p_pszSrcLocale, p_pszTrgLocale, p_pszCodeset);
		// Write header padding
		return(UpdateFile());
	}

	// Check the signature
	if ( fseek(m_pFile, 0, SEEK_SET) != 0 ) return(false);
	fread(szBuf, okLTM_LSIG, 1, m_pFile);
	szBuf[okLTM_LSIG] = 0;
	if ( strcmp(szBuf, okLTM_SIGNATURE) != 0 )
	{
		// Invalid format or wrong version.
		fclose(m_pFile);
		return(false);
	}

	// Source
	fread(szBuf, okLTM_LLOC, 1, m_pFile);
	m_sSrcLocale = szBuf;
	if ( p_pszSrcLocale != NULL )
	{
		if ( m_sSrcLocale != p_pszSrcLocale )
		{
			// The source locale of this TM is not the one expected
			fclose(m_pFile);
			return(false);
		}
	}

	// Target
	fread(szBuf, okLTM_LLOC, 1, m_pFile);
	m_sTrgLocale = szBuf;
	if ( p_pszTrgLocale != NULL )
	{
		if ( m_sTrgLocale != p_pszTrgLocale )
		{
			// The target locale of this TM is not the one expected
			fclose(m_pFile);
			return(false);
		}
	}

	// Code set
	fread(szBuf, okLTM_LCS, 1, m_pFile);
	m_sCodeset = szBuf;

	// Number of records
	fread(&m_lCount, sizeof(m_lCount), 1, m_pFile);

	// Index location
	fread(&m_lIndexPos, sizeof(m_lIndexPos), 1, m_pFile);

	// Index size
	fread(&m_lIndexSize, sizeof(m_lIndexSize), 1, m_pFile);

	// Allocate memory
	m_pIndex = (char*)malloc(m_lIndexSize);
	if ( m_pIndex == NULL )
	{
		// Could not allocate memory
		fclose(m_pFile);
		return(false);
	}

	// Read the index
	if ( fseek(m_pFile, m_lIndexPos, SEEK_SET) != 0 )
	{
		// Error
		fclose(m_pFile);
		return(false);
	}

	if ( m_lIndexSize )
		return(fread(m_pIndex, m_lIndexSize, 1, m_pFile) == 1);

	return(true);

#undef LBUF
}


/*--------------------------------------------------------------------------*/

bool okLiteTM::UpdateFile ()
{
#define LBUF 100

	char                 szBuf[LBUF+1];


	if ( m_bReadOnly ) return(false);
	if ( fseek(m_pFile, 0, SEEK_SET) != 0 ) return(false);


	//--- Write the header (overwrite the previous)

	// Write the version id
	fwrite(okLTM_SIGNATURE, okLTM_LSIG, 1, m_pFile);

	// Source
	memset(szBuf, 0, LBUF);
	strcpy(szBuf, m_sSrcLocale.c_str());
	fwrite(szBuf, okLTM_LLOC, 1, m_pFile);

	// Target
	memset(szBuf, 0, LBUF);
	strcpy(szBuf, m_sTrgLocale.c_str());
	fwrite(szBuf, okLTM_LLOC, 1, m_pFile);

	// Code set
	memset(szBuf, 0, LBUF);
	strcpy(szBuf, m_sCodeset.c_str());
	fwrite(szBuf, okLTM_LCS, 1, m_pFile);

	// Number of records
	fwrite(&m_lCount, sizeof(m_lCount), 1, m_pFile);

	// Index location
	fwrite(&m_lIndexPos, sizeof(m_lIndexPos), 1, m_pFile);

	// Index size
	fwrite(&m_lIndexSize, sizeof(m_lIndexSize), 1, m_pFile);


	//--- Write the Index (add at the end of the file)

	// Move to the new start of index position
	if ( fseek(m_pFile, m_lIndexPos, SEEK_SET) != 0 )
	{
		// Index position is out of bounds
		return(false);
	}

	// Write index data
	if ( m_lIndexSize )
		return(fwrite(m_pIndex, m_lIndexSize, 1, m_pFile) == 1);

	return(true);

#undef LBUF
}


/*--------------------------------------------------------------------------*/

bool okLiteTM::SaveAsOpenTag
(
	const char*          p_pszPath,
	const char*          p_pszTool,
	const bool           p_bOutputKeys
)
{
	long                 lPos;
	okLiteTMRecord       Record;
	FILE*                pFile;
	string               sTmp;
	string               sBuf;
	char                 szBuf[255];
	bool                 bRes = true;
	char*                psz;
	int                  nID = 1;


	if ( (pFile = fopen(p_pszPath, "wt")) == NULL )
	{
		// Could not open the file
		return(false);
	}

	// Write the header
	fprintf(pFile, "<?xml version='1.0' encoding='%s' ?>\n",
		m_sCodeset.c_str());

	sTmp = m_sPath;
	fprintf(pFile, "<opentag version='1.2'>\n <file tool='%s' "\
		"lc='%s' original='%s' datatype='plaintext'>\n",
		p_pszTool, m_sSrcLocale.c_str(), ConvertToXML(sTmp, true));

	lPos = MoveToFirstRecord();
	while ( lPos != okLTM_EOF )
	{
		// Read the data
		if ( !(bRes = Get(Record, lPos)) )
		{
			// Error reading the record
			break;
		}

		// Build the attribute for the group
		sBuf = "";
		if ( !Record.m_sId.empty() )
		{
			sTmp = Record.m_sId;
			sprintf(szBuf, " id='%s'", ConvertToXML(sTmp, true));
		}
		else
		{
			// No ID: create one
			// The id attribute is not required, but it is nice to have one
			sprintf(szBuf, " id='%d'", nID);
		}
		sBuf += szBuf;

		if ( !Record.m_sType.empty() )
		{
			sTmp = Record.m_sType;
			sprintf(szBuf, "type='%s'", ConvertToXML(sTmp, true));
			sBuf += " ";
			sBuf += szBuf;
		}
		if ( Record.m_nX > -1 )
		{
			sprintf(szBuf, " coord='%d;%d;%d;%d'", Record.m_nX, Record.m_nY,
				Record.m_nCX, Record.m_nCY);
			sBuf += " ";
			sBuf += szBuf;
		}

		// Write the data
		fprintf(pFile, "  <grp%s>\n", sBuf.c_str());

		// Index key (if requested)
		if ( p_bOutputKeys )
		{
			fputs("   <prop type='tmkey'>", pFile);
			sBuf = ""; psz = m_pCurPos+sizeof(long);
			while ( *psz )
			{
				sBuf += *psz;
				if ( !*(++psz) )
				{
					// Add a space (except for the last word)
					if ( *(++psz) ) sBuf += ' ';
				}
			}
			fprintf(pFile, "%s</prop>\n", sBuf.c_str());
		}

		// Source
		fprintf(pFile, "   <p lc='%s'>", m_sSrcLocale.c_str());
		sTmp = Record.m_sSource;
		fputs(ConvertToXML(sTmp, false), pFile);

		// Target
		fprintf(pFile, "</p>\n   <p lc='%s'>", m_sTrgLocale.c_str());
		sTmp = Record.m_sTarget;
		fputs(ConvertToXML(sTmp, false), pFile);
		fputs("</p>\n  </grp>\n", pFile);

		// Move to the next record
		lPos = MoveToNextRecord();
		nID++;
	}

	// Write trailer
	fprintf(pFile, " </file>\n</opentag>\n");
	fclose(pFile);
	return(bRes);
}


/*--------------------------------------------------------------------------*/
/* This will work only for iso-8859-1 text: we rely on the compatibility of
   the code set with Unicode to simply convert single-byte into wide
	Unicode characters. 
	Note that characters between 0x80 and 0x9F included will be replaced by
	question marks ('?') since they are not iso-8859-1 valid codes and
	therefore not Unicode compatible. */

bool okLiteTM::SaveAsTMX
(
	const char*          p_pszPath,
	const char*          p_pszTool,
	const char*          p_pszToolVersion,
	const bool           p_bOutputKeys
)
{
	long                 lPos;
	okLiteTMRecord       Record;
	FILE*                pFile;
	string               sTmp;
	string               sBuf;
	char                 szBuf[255];
	bool                 bRes = true;
	char*                psz;
	int                  nTUID = 1;


	if ( (pFile = fopen(p_pszPath, "wb")) == NULL )
	{
		// Could not open the file
		return(false);
	}

	// Write the header
	fputs("\xff\xfe", pFile);
	fwprintf(pFile, L"<?xml version='1.0' encoding='iso-10646-ucs-2' ?>\r\n");
	fputws(L"<!DOCTYPE tmx SYSTEM 'tmx11.dtd'>\r\n", pFile);

	fputws(L"<tmx version='1.1'>\r\n", pFile);

	// Header

	fwprintf(pFile, L" <header\r\n  creationtool='%S'\r\n", p_pszTool);
	fwprintf(pFile, L"  creationtoolversion='%S'\r\n  datatype='plaintext'\r\n",
		p_pszToolVersion);
	fputws(L"  segtype='paragraph'\r\n  adminlang='EN-US'\r\n", pFile);
	fwprintf(pFile, L"  srclang='%S'\r\n  o-tmf='okLiteTM'\r\n",
		m_sSrcLocale.c_str());
	fputws(L" >\r\n </header>\r\n <body>\r\n", pFile);

	lPos = MoveToFirstRecord();
	while ( lPos != okLTM_EOF )
	{
		// Read the data
		if ( !(bRes = Get(Record, lPos)) )
		{
			// Error reading the record
			break;
		}

		// Build the attribute for the tu element
		sBuf = "";
		if ( !Record.m_sId.empty() )
		{
			sTmp = Record.m_sId;
			sprintf(szBuf, "tuid='%s'", ConvertToXML(sTmp, true, true));
		}
		else
		{
			// No ID for the record: create a default one
			// The tuid attribute is not required, but it's nice to have one
			sprintf(szBuf, "tuid='%d'", nTUID);
		}
		sBuf += szBuf;

		// Write the data
		fwprintf(pFile, L"  <tu %S>\r\n", sBuf.c_str());

		// Properties
		// Index key (if requested)
		if ( p_bOutputKeys )
		{
			fputws(L"   <prop type='tmkey'>", pFile);
			sBuf = ""; psz = m_pCurPos+sizeof(long);
			while ( *psz )
			{
				sBuf += *psz;
				if ( !*(++psz) )
				{
					// Add a space (except for the last word)
					if ( *(++psz) ) sBuf += ' ';
				}
			}
			fwprintf(pFile, L"%S</prop>\r\n", sBuf.c_str());
		}

		// Coordinates (if any)
		if ( Record.m_nX != -1 )
		{
			fputws(L"   <prop type='coord'>", pFile);
			sprintf(szBuf, "%d;%d;%d;%d", Record.m_nX, Record.m_nY,
				Record.m_nCX, Record.m_nCY);
			fwprintf(pFile, L"%S</prop>\r\n", szBuf);
		}

		// Type (if any)
		if ( !Record.m_sType.empty() )
		{
			fputws(L"   <prop type='type'>", pFile);
			sTmp = Record.m_sType;
			fwprintf(pFile, L"%S</prop>\r\n",
				ConvertToXML(sTmp, false, true));
		}

		fwprintf(pFile, L"   <tuv lang='%S'>\r\n    <seg>",
			m_sSrcLocale.c_str());

		// Source
		sTmp = Record.m_sSource;
		fwprintf(pFile, L"%S", ConvertToXML(sTmp, false, true));

		// Target
		fwprintf(pFile, L"</seg>\r\n   </tuv>\r\n   <tuv lang='%S'>\r\n",
			m_sTrgLocale.c_str());
		sTmp = Record.m_sTarget;
		fwprintf(pFile, L"    <seg>%S", ConvertToXML(sTmp, false, true));
		fputws(L"</seg>\r\n   </tuv>\r\n  </tu>\r\n", pFile);

		// Move to the next record
		lPos = MoveToNextRecord();
		nTUID++;
	}

	// Write trailer
	fputws(L" </body>\r\n</tmx>\r\n", pFile);
	fclose(pFile);
	return(bRes);
}


/*--------------------------------------------------------------------------*/

bool okLiteTM::MakeKey
(
	const char*          p_pszText,
	char**               p_ppData,
	int*                 p_pnLength
)
{
	bool                 bWasSpace = false;
	string               sTmp = "";
	int                  i = 0;
	int                  nExtraLen = okLTM_STARTKEY+2;
	char                 ch;

	
	// Remove noise characters and convert to lowercases
	// We assume ASCII-based and single-bye code set here
	while ( (ch = *(p_pszText+i)) )
	{
		i++;

		if ( isspace(ch) )
		{
			if ( !bWasSpace )
			{
				if ( sTmp.empty() ) continue; // No leading spaces
				bWasSpace = true;
				continue;
			}
		}

		switch ( ch )
		{
			case 0x7b:
			case 0x7c:
			case 0x7e:
			case 0x7f:
			case 0x5b:
			case 0x5c:
			case 0x5e:
			case 0x5f:
				// Treat it like a space
				if ( !bWasSpace )
				{
					if ( sTmp.empty() ) continue; // No leading spaces
					bWasSpace = true;
					continue;
				}
				// Noise characters: skip over
				continue;

			default:
				if ( ch < 0x41 ) // Non-alpha characters
				{
					// Remove '&' (better result than replacing with space)
					if ( ch == 0x26 ) continue;
					// Other: treat them like spaces
					if ( !bWasSpace )
					{
						if ( sTmp.empty() ) continue; // No leading spaces
						bWasSpace = true;
						continue;
					}
					continue;
				}
				if ( bWasSpace )
				{
					// Unique space
					bWasSpace = false;
					sTmp += 0x20;
				}
				// We assume ASCII text here
				sTmp += tolower(ch);
				continue;
		}
	}

	if ( sTmp.empty() )
	{
		*p_pnLength = 0;
		return(true);
	}

	// Allocate the new key
	*p_pnLength = sTmp.length()+nExtraLen; // Length + 2 zero + 1 long
	*p_ppData = new char[*p_pnLength];
	if ( *p_ppData == NULL )
	{
		return(false);
	}

	// Copy the key
	memset(*p_ppData, 0,  *p_pnLength);
	memcpy(*p_ppData+okLTM_STARTKEY, sTmp.c_str(), (*p_pnLength)-nExtraLen);

	// Replace spaces by null characters
	for ( i=sizeof(long); i<((*p_pnLength)-2); i++ )
		if ( *((*p_ppData)+i) == 0x20 ) *((*p_ppData)+i) = 0;

	return(true);
}


/*--------------------------------------------------------------------------*/
// Returns okLTM_ADDED, okLTM_EMPTY, okLTM_DUPLICATE or okLTM_ADD_ERROR

int okLiteTM::Add
(
	const char*          p_pszSource,
	const char*          p_pszTarget,
	const char*          p_pszType,
	const char*          p_pszId,
	const int            p_nX,
	const int            p_nY,
	const int            p_nCX,
	const int            p_nCY
)
{
	char*                pData;
	long                 lNewLen;
	int                  nLen;
	void*                pNewIndex;
	okLiteTMResultList   lstRes;
	okLiteTMResultIter   Iter;
	okLiteTMRecord       Record;
	

	//--- Add to the index

	if ( !MakeKey(p_pszSource, &pData, &nLen) ) return(okLTM_ADD_ERROR);
	if ( !nLen )
	{
		// No allocation of pData -> no need for delete()
		return(okLTM_ADD_EMPTY);
	}

	// Check for duplicate
	if ( Query(p_pszSource, 100, false, &lstRes) )
	{
		// Same source. What about the target?
		Iter = lstRes.begin();
		while ( 1 )
		{
			if ( Get(Record, Iter->m_lPos) )
			{
				if ( Record.m_sTarget == p_pszTarget )
				{
					// Same target. What about the type?
					if ( Record.m_sType == p_pszType )
					{
						// It's a duplicate: do not add
						delete[] pData;
						return(okLTM_ADD_DUPLICATE);
					}
				}
			}
			Iter++;
			if ( Iter == lstRes.end() ) break;
		}
		
	}

	// Set the position of the info on disk in the index key
	memcpy(pData, &m_lIndexPos, sizeof(long));

	// Add the new key to the index
	lNewLen = m_lIndexSize + nLen;
	pNewIndex = realloc((void*)m_pIndex, lNewLen);
	if ( pNewIndex == NULL )
	{
		delete[] pData;
		return(okLTM_ADD_ERROR);
	}
	m_pIndex = (char*)pNewIndex;
	memcpy(m_pIndex+m_lIndexSize, pData, nLen);
	m_lIndexSize = lNewLen;
	delete[] pData;


	//--- Add the data to the file

	// Add at the end of the data block, over the index
	if ( fseek(m_pFile, m_lIndexPos, SEEK_SET) != 0 ) return(okLTM_ADD_ERROR);

	Record.m_sSource = p_pszSource;
	Record.m_sTarget = p_pszTarget;
	Record.m_sType = p_pszType;
	Record.m_sId = p_pszId;
	Record.m_nX = p_nX;
	Record.m_nY = p_nY;
	Record.m_nCX = p_nCX;
	Record.m_nCY = p_nCY;

	if ( !Record.Write(m_pFile, m_lIndexPos) )
	{
		return(okLTM_ADD_ERROR);
	}

	// Update the counter
	m_lCount++;

	return(okLTM_ADD_ADDED);
}


/*--------------------------------------------------------------------------*/

int okLiteTM::Query
(
	const char*          p_pszQuery,
	const int            p_nThreshold,
	const bool           p_bConcordance,
	okLiteTMResultList*  p_pResults
)
{
	float                fThreshold = (p_nThreshold*0.01);
	char*                pszData;
	char*                pszD;
	char*                pszQ;
	char*                pQueryKey = NULL;
	char*                pQueryStart;
	int                  nQuerySize;
	int                  nFound;
	int                  nTotal = 0;
	int                  nCount = 0;
	int                  nInKey;
	okLiteTMRecord       Record;
	okLiteTMResult       Res;
	bool                 bSequence;


	if ( !m_lIndexSize ) return(0);

	//--- Convert the query into index key

	if ( !MakeKey(p_pszQuery, &pQueryKey, &nQuerySize) ) return(0);
	if ( !nQuerySize ) return(0);

	pQueryStart = pQueryKey+okLTM_STARTKEY;

	// Get the number of words in the query key
	pszQ = pQueryStart;
	while ( *pszQ )
	{
		nTotal++;
		while ( *pszQ ) pszQ++;
		pszQ++;
	}

	if ( nTotal <= 0 )
	{
		delete[] pQueryKey;
		return(0);
	}


	//--- Preform the query

	pszData = m_pIndex;
	while ( 1 )
	{
		// Get record position
		memcpy(&Res.m_lPos, pszData, sizeof(long));
		pszData += okLTM_STARTKEY;

		pszQ = pQueryStart;
		nFound = 0;
		bSequence = true;

		// for each word in the query
		while ( *pszQ )
		{
			// Search for the current query word in the key words
			pszD = pszData;
			nInKey = 0;

			// For each key
			while ( *pszD )
			{
				nInKey++;

				if ( strcmp(pszQ, pszD) == 0 )
				{
					nFound++;
					if ( bSequence )
					{
						if ( nInKey > nFound ) bSequence = false;
						else if ( nInKey < nFound )
						{
							// Skip this found
							nFound--;
							goto label_nextD;
						}
					}
					break;
				}
				// Else: move to the next word of the key
				label_nextD:
				while ( *pszD ) pszD++;
				pszD++; // Skip ending zero
			}

			// Move to the next word of the query
			while ( *pszQ ) pszQ++;
			pszQ++; // Skip ending zero
		}

		// Count the words in the record and move to the next one
		nInKey = 0;
		while ( *pszData )
		{
			while ( *pszData ) pszData++;
			pszData++; nInKey++;
		}
		pszData++;


		// Calculate the score
		if ( p_bConcordance )
		{
			// For concordance searches
			Res.m_fScore = (float)(nFound) / (float)(nTotal);
		}
		else
		{
			// For normal lookups (compensate for sub-strings matches)
			Res.m_fScore = (float)(2*nFound) / (float)(nTotal+nInKey);
		}

		// Store the length for secondary sort later
		Res.m_nLen = nInKey;

		
		//--- Adjust score

		if ( Res.m_fScore > 1.0 )
		{
			// Case where query has several time the same word that is one or
			// more time in the query.
			// A better solution is to flag the words found in the key while
			// doing the query to not re-match on them

			Res.m_fScore = 1.0; // Reduce it to 100
		}

		if ( Res.m_fScore == 1.0 )
		{
			if ( Get(Record, Res.m_lPos) )
			{
				// Text different
				if ( Record.m_sSource != p_pszQuery )
					Res.m_fScore -= (float)0.01;
				// Not sequential (for sub-strings)
				if ( !bSequence )
					Res.m_fScore -= (float)0.01;
			}
			else Res.m_fScore = 0.0; // Cannot access the record
		}

		// Is it a useable result
		if ( Res.m_fScore >= fThreshold )
		{
			// The match scores >= to the limit: add it to the list
			nCount++;
			p_pResults->push_back(Res);
		}

		// Stop when we reach the end of the index
		if ( pszData >= m_pIndex+m_lIndexSize ) break;
	}

	// Clean up
	if ( pQueryKey != NULL ) delete[] pQueryKey;

	// Return the number of matches found
	return(nCount);
}


/*--------------------------------------------------------------------------*/

long okLiteTM::MoveToFirstRecord ()
{
	long                 lPos;


	m_lCurIndex = 0;
	if ( !m_lCount ) return(okLTM_EOF); // Empty TM

	m_pCurPos = m_pIndex;
	memcpy(&lPos, m_pCurPos, sizeof(long));
	return(lPos);
}
	

/*--------------------------------------------------------------------------*/

long okLiteTM::MoveToNextRecord ()
{
	long                 lPos;


	if ( !m_lCount ) return(okLTM_EOF); // Empty TM

	m_pCurPos += okLTM_STARTKEY; // Skip the position

	// Skip the each words in the key
	while ( *m_pCurPos || *(m_pCurPos+1) ) m_pCurPos++;
	m_pCurPos += 2; // Skip the last zeros at the end of the key

	// Check if it is the end of the index
	if ( m_pCurPos >= m_pIndex+m_lIndexSize ) return(okLTM_EOF);
	
	// Get the position of the new current record
	memcpy(&lPos, m_pCurPos, sizeof(long));
	m_lCurIndex++;
	return(lPos);
}
	

/*--------------------------------------------------------------------------*/

bool okLiteTM::Get
(
	okLiteTMRecord&      p_rRecord,
	const long           p_lPos
)
{
	if ( fseek(m_pFile, p_lPos, SEEK_SET) != 0 ) return(false);
	return(p_rRecord.Read(m_pFile));
}



/*--------------------------------------------------------------------------*/
/* Control characters (characters below 0x20 (space)) are automatically
   converted to '?' to avoid problems later. You should handle them more
   appropriately if the routine is to be used for something else than demo. */

bool okLiteTM::AddEntriesFromFile
(
	const char*          p_pszPath,
	int*                 p_pnTotal,
	int*                 p_pnAdded,
	int*                 p_pnEmpty,
	int*                 p_pnDuplicate,
	int*                 p_pnError
)
{
#define LBUF 2048

	FILE*                pFile;
	char                 szBuf[LBUF+1];
	char*                psz;
	string               sSource;
	string               sTarget;
	string               sID;
	string               sType;
	int                  i;
	int                  nLen;


	*p_pnTotal = 0;
	*p_pnAdded = 0;
	*p_pnEmpty = 0;
	*p_pnDuplicate = 0;
	*p_pnError = 0;

	if ( (pFile = fopen(p_pszPath, "rt")) == NULL )
	{
		// Could not open the file
		return(false);
	}

	// Each line: Source<tab>Target[<tab>ID[<tab>Type]]<crlf>

	while ( 1 )
	{
		if ( fgets(szBuf, LBUF, pFile) == NULL ) break;
		szBuf[LBUF] = 0;

		// Replace Control chars by '?'
		for ( i=0, nLen=strlen(szBuf); i<nLen; i++ )
		{
			if (( ((unsigned char)szBuf[i]) < 0x20 ) &&
					(( szBuf[i] != '\t' ) && ( szBuf[i] != '\n' )))
			{
				szBuf[i] = '?';
			}
		}

		// Source text
		psz = strtok(szBuf, "\t\n");
		if ( psz == NULL ) sSource = "";
		else sSource = psz;

		// Target text
		psz = strtok(NULL, "\t\n");
		if ( psz == NULL ) sTarget = "";
		else sTarget = psz;

		// ID
		psz = strtok(NULL, "\t\n");
		if ( psz == NULL ) sID = "";
		else sID = psz;

		// Type
		psz = strtok(NULL, "\t\n");
		if ( psz == NULL ) sType = "";
		else sType = psz;

		(*p_pnTotal)++;
		switch ( Add(sSource.c_str(), sTarget.c_str(), sType.c_str(),
			sID.c_str()) )
		{
			case okLTM_ADD_ERROR:
				(*p_pnError)++;
				break;
			case okLTM_ADD_ADDED:
				(*p_pnAdded)++;
				break;
			case okLTM_ADD_EMPTY:
				(*p_pnEmpty)++;
				break;
			case okLTM_ADD_DUPLICATE:
				(*p_pnDuplicate)++;
				break;
		}

	}

	// Save the updated index
	UpdateFile();

	fclose(pFile);
	return(true);

#undef LBUF
}

