{ This example unit shows how to implement a custom FTS3 / FTS2 tokenizer
  with DISQLite3.

  This tokenizer uses proper Pascal delimiters (the '_' underscore is NOT a
  delimiter), decodes UTF-8 input, and outputs UTF-8 tokens.

  Visit the DISQLite3 Internet site for latest information and updates:

    http://www.yunqa.de

  Copyright (c) 2005-2021 Ralf Junker, Yunqa <delphi@yunqa.de>

------------------------------------------------------------------------------ }

{ }
unit DISQLite3Tokenizer;

{$I DICompilers.inc}
{$I DISQLite3.inc}

interface

uses
  DISystemCompat,
  {$IFDEF HAS_UNITSCOPE}
  System.SysUtils,
  {$ELSE HAS_UNIT_SCOPE}
  SysUtils,
  {$ENDIF HAS_UNIT_SCOPE}
  {$IFDEF test_}DISQLite3Types, {$ENDIF}DISQLite3Api
  {$IFDEF YuStemmer}, YuStemmer{$ENDIF}; // Download from http://www.yunqa.de.

{ Returns a pointer to the Unicode Tokenizer Module. }
function sqlite3Fts3UnicodeTokenizerModule: sqlite3_tokenizer_module_ptr;

{ Returns a pointer to the Pascal Tokenizer Module. }
function sqlite3Fts3PascalTokenizerModule: sqlite3_tokenizer_module_ptr;

implementation

uses
  {$IFDEF DIH_HAVE_DILibC}DILibC, {$ENDIF}DISQLite3TokenizerUtils;

type

  { Structures used by the tokenizer interface. When a new tokenizer
    implementation is registered, the caller provides a pointer to
    an sqlite3_tokenizer_module containing pointers to the callback
    functions that make up an implementation.

    When an fts3 table is created, it passes any arguments passed to
    the tokenizer clause of the CREATE VIRTUAL TABLE statement to the
    sqlite3_tokenizer_module.xCreate() function of the requested tokenizer
    implementation. The xCreate() function in turn returns an
    sqlite3_tokenizer structure representing the specific tokenizer to
    be used for the fts3 / fts2 table (customized by the tokenizer clause
    arguments).

    To tokenize an input buffer, the sqlite3_tokenizer_module.xOpen()
    method is called. It returns an sqlite3_tokenizer_cursor object
    that may be used to tokenize a specific input buffer based on
    the tokenization rules supplied by a specific sqlite3_tokenizer
    object. }

  { }
  unicode_tokenizer = record
    Base: sqlite3_tokenizer;
    {$IFDEF YuStemmer}
    Stemmer: TYuStemmer_8;
    {$ENDIF YuStemmer}
  end;
  unicode_tokenizer_ptr = ^unicode_tokenizer;

  { }
  unicode_tokenizer_cursor = record
    Base: sqlite3_tokenizer_cursor;
    pInput: PUtf8Char; // input where we are tokenizing.
    pPos: PUtf8Char; // current position.
    nBytes: Cardinal; // input remaining at pPos.
    iOffset: Integer; // current position in pInput.
    iToken: Integer; // index of next token to be returned.
    pToken: PUtf8Char; // storage for current token.
    nTokenAllocated: Integer; // space allocated to zToken buffer.
    CharIsDelim: function(const Char: Cardinal): Boolean;
    CharToLower: function(const Char: Cardinal): Cardinal;
    {$IFDEF YuStemmer}
    Stemmer: TYuStemmer_8;
    {$ENDIF YuStemmer}
  end;
  unicode_tokenizer_cursor_ptr = ^unicode_tokenizer_cursor;

  //------------------------------------------------------------------------------

{ Create a new tokenizer. The values in the argv[] array are the
  arguments passed to the "tokenizer" clause of the CREATE VIRTUAL
  TABLE statement that created the fts3 / fts2 table. For example, if
  either of the following SQL is executed:

    CREATE .. USING fts3( ... , tokenizer <tokenizer-name> arg1 arg2);
    CREATE .. USING fts2( ... , tokenizer <tokenizer-name> arg1 arg2);

  then argc is set to 2, and the argv[] array contains pointers
  to the strings "arg1" and "arg2".

  This method should return either SQLITE_OK, or an SQLite error
  code. If SQLITE_OK is returned, then ppTokenizer should be set
  to point at the newly created tokenizer structure. The generic
  sqlite3_tokenizer.pModule variable should not be initialised by
  this callback. The caller will do so. }

function unicode_tokenizer_Create(
  argc: Integer; // Number of entries in argv[].
  const argv: PPUtf8CharArray; // Tokenizer creation arguments.
  ppTokenizer: sqlite3_tokenizer_ptr_ptr // OUT: Created tokenizer.
  ): Integer; {$IFDEF DISQLite3_CDECL} cdecl; {$ENDIF}
var
  {$IFDEF YuStemmer}
  Stemmer: TYuStemmer_8;
  {$ENDIF YuStemmer}
  t: unicode_tokenizer_ptr;
begin
  {$IFDEF YuStemmer}
  if argc > 0 then
    begin
      Stemmer := GetStemmer_8(argv[0]);
      { If creating the stemmer failed (wrong algorithm name, for example),
        return an error. }
      if not Assigned(Stemmer) then
        begin
          Result := SQLITE_ERROR;
          Exit;
        end;
    end
  else
    Stemmer := nil;
  {$ELSE YuStemmer}
  { No stemmer: Report any unrecognized arguments as error. }
  if argc > 0 then
    begin
      Result := SQLITE_ERROR;
      Exit;
    end;
  {$ENDIF YuStemmer}

  try
    GetMem(t, SizeOf(t^));
  except
    Result := SQLITE_NOMEM;
    Exit;
  end;

  {$IFDEF YuStemmer}
  t^.Stemmer := Stemmer;
  {$ENDIF YuStemmer}
  ppTokenizer^ := Pointer(t);
  Result := SQLITE_OK;
end;

//------------------------------------------------------------------------------

{ Destroy an existing tokenizer. The fts3 / fts2 module calls this method
  exactly once for each successful call to xCreate(). }

function unicode_tokenizer_destroy(
  pTokenizer: sqlite3_tokenizer_ptr
  ): Integer; {$IFDEF DISQLite3_CDECL} cdecl; {$ENDIF}
begin
  {$IFDEF YuStemmer}
  unicode_tokenizer_ptr(pTokenizer)^.Stemmer.Free;
  {$ENDIF YuStemmer}
  FreeMem(pTokenizer);
  Result := SQLITE_OK;
end;

//------------------------------------------------------------------------------

{ Create a tokenizer cursor to tokenize an input buffer. The caller
  is responsible for ensuring that the input buffer remains valid
  until the cursor is closed (using the xClose() method). }

function unicode_tokenizer_Open(
  pTokenizer: sqlite3_tokenizer_ptr; // The tokenizer
  const pInput: PUtf8Char; // Input string
  nBytes: Integer; // Length of pInput in bytes
  ppCursor: sqlite3_tokenizer_cursor_ptr_ptr // OUT: Tokenization cursor
  ): Integer; {$IFDEF DISQLite3_CDECL} cdecl; {$ENDIF}
var
  c: unicode_tokenizer_cursor_ptr;
begin
  try
    GetMem(c, SizeOf(c^));
  except
    Result := SQLITE_NOMEM;
    Exit;
  end;
  c^.pInput := pInput;
  c^.pPos := pInput;
  if Assigned(pInput) then
    if nBytes < 0 then
      c^.nBytes := {$IFNDEF DISQLite3_DLL}{$IFDEF DIH_HAVE_DILibC}DILibC{$ELSE}DISQLite3Api{$ENDIF}.{$ENDIF}StrLen(pInput)
    else
      c^.nBytes := nBytes
  else
    c^.nBytes := 0;

  c^.iToken := 0;
  c^.pToken := nil; // No space allocated, yet.
  c^.nTokenAllocated := 0;

  c^.CharIsDelim := CharIsDelimiter;
  c^.CharToLower := CharToCaseFold;

  {$IFDEF YuStemmer}
  c^.Stemmer := unicode_tokenizer_ptr(pTokenizer)^.Stemmer;
  {$ENDIF YuStemmer}

  ppCursor^ := Pointer(c);
  Result := SQLITE_OK;
end;

//------------------------------------------------------------------------------

{ Destroy an existing tokenizer cursor. The fts3 / fts2 module calls this
  method exactly once for each successful call to xOpen(). }

function unicode_tokenizer_Close(
  pCursor: sqlite3_tokenizer_cursor_ptr): Integer; {$IFDEF DISQLite3_CDECL} cdecl; {$ENDIF}
var
  c: unicode_tokenizer_cursor_ptr;
begin
  c := Pointer(pCursor);
  FreeMem(c^.pToken);
  FreeMem(c);
  Result := SQLITE_OK;
end;

//------------------------------------------------------------------------------

{ Retrieve the next token from the tokenizer cursor pCursor. This
  method should either return SQLITE_OK and set the values of the
  "OUT" variables identified below, or SQLITE_DONE to indicate that
  the end of the buffer has been reached, or an SQLite error code.

  ppToken should be set to point at a buffer containing the
  normalized version of the token (i.e. after any case-folding and/or
  stemming has been performed). pnBytes should be set to the length
  of this buffer in bytes. The input text that generated the token is
  identified by the byte offsets returned in piStartOffset and
  piEndOffset.

  The buffer ppToken is set to point at is managed by the tokenizer
  implementation. It is only required to be valid until the next call
  to xNext() or xClose(). }

function unicode_tokenizer_Next(
  pCursor: sqlite3_tokenizer_cursor_ptr; // Cursor returned by unicode_tokenizer_Open.
  const ppToken: PPUtf8Char; // OUT: ppToken is the token text.
  pnBytes: PInteger; // OUT: Number of bytes in token.
  piStartOffset: PInteger; // OUT: Starting offset of token.
  piEndOffset: PInteger; // OUT: Ending offset of token.
  piPosition: PInteger // OUT: Position integer of token.
  ): Integer; {$IFDEF DISQLite3_CDECL} cdecl; {$ENDIF}
label
  lblToken;
var
  c: unicode_tokenizer_cursor_ptr;
  Ch: Cardinal;
  pTextPos, pTokenPos: PUtf8Char;
  l, TokenRemaining, TokenUsed: Integer;
begin
  c := Pointer(pCursor);

  // Find the first non-delimiter character     .
  while c^.nBytes > 0 do
    begin
      l := sqlite3_read_utf8(c^.pPos, c^.nBytes, Ch);
      if not c^.CharIsDelim(Ch) then
        goto lblToken;
      Inc(c^.pPos, l); Dec(c^.nBytes, l);
    end;

  Result := SQLITE_DONE;
  Exit;

  l := 0; // Silence compiler warning.

  lblToken:
  { At this point, Ch holds a non-delimiter token character. Write it to the
    token buffer and continue reading the token up to the next delimiter or
    until all input is consumed. }
  pTextPos := c^.pPos;
  pTokenPos := c^.pToken;
  TokenRemaining := c^.nTokenAllocated;

  repeat
    Inc(c^.pPos, l); Dec(c^.nBytes, l);

    { Make sure the token buffer is big enough to hold a token character.
      UTF-8 writes at most 6 bytes. }
    if TokenRemaining < 6 then
      begin
        TokenUsed := pTokenPos - c^.pToken;
        Inc(c^.nTokenAllocated, $20);
        ReallocMem(c^.pToken, c^.nTokenAllocated);
        pTokenPos := c^.pToken + TokenUsed;
        TokenRemaining := c^.nTokenAllocated - TokenUsed;
      end;

    Ch := c^.CharToLower(Ch);
    l := sqlite3_write_utf8(Ch, pTokenPos);
    Inc(pTokenPos, l); Dec(TokenRemaining, l);

    if c^.nBytes <= 0 then
      Break;
    l := sqlite3_read_utf8(c^.pPos, c^.nBytes, Ch);
  until c^.CharIsDelim(Ch);

  l := pTokenPos - c^.pToken; // Length of extracted token.

  ppToken^ := c^.pToken;
  pnBytes^ := l;
  {$IFDEF YuStemmer}
  if Assigned(c^.Stemmer) then
    c^.Stemmer.Stem(ppToken^, pnBytes^);
  {$ENDIF YuStemmer}

  piStartOffset^ := pTextPos - c^.pInput;
  piEndOffset^ := c^.pPos - c^.pInput;

  piPosition^ := c^.iToken;
  Inc(c^.iToken);
  Result := SQLITE_OK;
end;

//------------------------------------------------------------------------------

{ Returns the Unicode tokenizer module. }

const
  unicode_tokenizer_module: sqlite3_tokenizer_module = (
    iVersion: 0;
    xCreate: unicode_tokenizer_Create;
    xDestroy: unicode_tokenizer_destroy;
    xOpen: unicode_tokenizer_Open;
    xClose: unicode_tokenizer_Close;
    xNext: unicode_tokenizer_Next);

function sqlite3Fts3UnicodeTokenizerModule: sqlite3_tokenizer_module_ptr;
begin
  Result := @unicode_tokenizer_module;
end;

//------------------------------------------------------------------------------
// Pascal Tokenizer
//------------------------------------------------------------------------------

{ Helper Function: Determine if a Unicode Code Point qualifies as a Pascal
  delimiter. Handles Latin 1 (ISO-8859-1) characters only (for size). }

function CharIsPascalDelimiter(const c: Cardinal): Boolean;
begin
  Result := CharIsDelimiter(c) and (c <> $5F { = '_' Underscore });
end;

//------------------------------------------------------------------------------

{ Overwrites unicode_tokenizer_Open and sets Pascal delimiter callback. }

function pascal_tokenizer_Open(
  pTokenizer: sqlite3_tokenizer_ptr; // The tokenizer
  const pInput: PUtf8Char; // Input string
  nBytes: Integer; // Length of pInput in bytes
  ppCursor: sqlite3_tokenizer_cursor_ptr_ptr // OUT: Tokenization cursor
  ): Integer; {$IFDEF DISQLite3_CDECL} cdecl; {$ENDIF}
var
  c: unicode_tokenizer_cursor_ptr;
begin
  Result := unicode_tokenizer_Open(pTokenizer, pInput, nBytes, ppCursor);
  c := Pointer(ppCursor^);
  c^.CharIsDelim := CharIsPascalDelimiter;
end;

//------------------------------------------------------------------------------

{ Returns the Pascal tokenizer module. }

const
  pascal_tokenizer_module: sqlite3_tokenizer_module = (
    iVersion: 0;
    xCreate: unicode_tokenizer_Create;
    xDestroy: unicode_tokenizer_destroy;
    xOpen: pascal_tokenizer_Open;
    xClose: unicode_tokenizer_Close;
    xNext: unicode_tokenizer_Next);

function sqlite3Fts3PascalTokenizerModule: sqlite3_tokenizer_module_ptr;
begin
  Result := @pascal_tokenizer_module;
end;

end.

