unit XData.Query.Tokenizer;

{$I XData.inc}

interface

uses
  System.SysUtils, System.Character;

type
  TQueryTokenType = (Identifier, OpenParen, CloseParen, Slash, Comma, Dot, Negate, EOF,
    TextLiteral, IntegerLiteral, Int64Literal, DoubleLiteral, BooleanLiteral, NullLiteral, DateTimeLiteral, GuidLiteral);

  TQueryTokenizer = class
  strict private
    // token info
    FIntTokenType: TQueryTokenType;
    FIdValue: string;
    FStringValue: string;
    FInt64Value: Int64;
    FIntegerValue: Integer;
    FDoubleValue: double;
    FBooleanValue: boolean;
    FDateTimeValue: TDateTime;
    FGuidValue: TGuid;
  strict private
    FExpr: string;
    FPos: integer;
    FLastPos: integer;
    FLen: integer;
    FFormatSettings: TFormatSettings;
    function NextChar: Char; inline;
    function PeekChar: Char; inline;
    function EOF: boolean; inline;
    procedure MoveNext; inline;
    function IsDigit(C: Char): boolean; inline;
    function IsLetter(C: Char): boolean; inline;
    function IsWhitespace(const C: Char): boolean; inline;
    function SkipWhitespace: Char;
    function IsIdentifierStart(const C: Char): boolean; inline;
    function IsIdentifierPart(const C: Char): boolean; inline;
    function ExtractString: string;
  strict private
    procedure ParseIdentifier;
    procedure ParseBinary;
    function TryParseDateTime(TokenPos: integer): boolean;
    function TryParseGuid(TokenPos: integer): boolean;
    function ParseLiteral(TokenPos: integer): string;
    function ParseFromDigit: TQueryTokenType;
    function IsNumeric(TokenType: TQueryTokenType): boolean;
    procedure ValidateDigit;
  public
    constructor Create(const AExpression: string);
    function NextToken: TQueryTokenType;
  public
    property TokenType: TQueryTokenType read FIntTokenType;
    property IdValue: string read FIdValue;
    property StringValue: string read FStringValue;
    property Int64Value: Int64 read FInt64Value;
    property IntegerValue: Integer read FIntegerValue;
    property DoubleValue: double read FDoubleValue;
    property BooleanValue: boolean read FBooleanValue;
    property DateTimeValue: TDateTime read FDateTimeValue;
    property GuidValue: TGuid read FGuidValue;
    property LastPosition: integer read FLastPos;
  end;

implementation

uses
  Bcl.Utils,
  Sparkle.Utils,
  XData.Query.Exceptions,
  XData.Utils;

{ TXDataQueryTokenizer }

constructor TQueryTokenizer.Create(const AExpression: string);
begin
  TBclUtils.InitFormatSettings(FFormatSettings);
  FFormatSettings.DecimalSeparator := '.';
  FExpr := AExpression;
  FPos := 1;
  FLen := Length(FExpr);
end;

function TQueryTokenizer.EOF: boolean;
begin
  Result := FPos > FLen;
end;

function TQueryTokenizer.ExtractString: string;
var
  C: Char;
begin
  Result := '';
  NextChar; // Skip quote
  repeat
    C := NextChar;
    while (C <> #0) and (C <> '''') do
    begin
      Result := Result + C;
      C := NextChar;
    end;

    if C = #0 then
      raise EQueryUnterminatedStringError.Create(Self)
    else
    begin
      if PeekChar = '''' then // check for double quotes
      begin
        NextChar;
        Result := Result + '''';
      end
      else
        break;
    end;
  until false;
end;

function TQueryTokenizer.IsDigit(C: Char): boolean;
begin
  Result := (Ord(C) <= 255) and (byte(C) in [Ord('0')..Ord('9')]);
end;

function TQueryTokenizer.IsIdentifierPart(const C: Char): boolean;
begin
  {$IFDEF DELPHIXE4_LVL}
  Result := C.IsLetterOrDigit or (C = '_');
  {$ELSE}
  Result := System.Character.IsLetterOrDigit(C) or (C = '_');
  {$ENDIF}
end;

function TQueryTokenizer.IsIdentifierStart(const C: Char): boolean;
begin
  {$IFDEF DELPHIXE4_LVL}
  Result := C.IsLetter or (C = '_') or (C = '$');
  {$ELSE}
  Result := System.Character.IsLetter(C) or (C = '_') or (C = '$');
  {$ENDIF}
end;

function TQueryTokenizer.IsLetter(C: Char): boolean;
begin
  {$IFDEF DELPHIXE4_LVL}
  Result := C.IsLetter;
  {$ELSE}
  Result := System.Character.IsLetter(C);
  {$ENDIF}
end;

function TQueryTokenizer.IsNumeric(TokenType: TQueryTokenType): boolean;
begin
  Result := TokenType in [TQueryTokenType.IntegerLiteral, TQueryTokenType.Int64Literal, TQueryTokenType.DoubleLiteral];
end;

function TQueryTokenizer.IsWhitespace(const C: Char): boolean;
begin
  Result := (Ord(C) <= 32) and (byte(C) in [$20, $A, $D, $9, $C]);
end;

procedure TQueryTokenizer.MoveNext;
begin
  Inc(FPos);
end;

function TQueryTokenizer.NextChar: Char;
begin
  if not EOF then
  begin
    Result := FExpr[FPos];
    Inc(FPos);
  end else
    Result := #0;
end;

function TQueryTokenizer.NextToken: TQueryTokenType;
var
  C: Char;
  TokenText: string;
  TokenLastPos: integer;
  _type: TQueryTokenType;
begin
  C := SkipWhitespace;
  FLastPos := FPos;
  TokenLastPos := FPos;
  case C of
    '(':
      begin
        _type := TQueryTokenType.OpenParen;
        NextChar;
      end;
    ')':
      begin
        _type := TQueryTokenType.CloseParen;
        NextChar;
      end;
    '/':
      begin
        _type := TQueryTokenType.Slash;
        NextChar;
      end;
    ',':
      begin
        _type := TQueryTokenType.Comma;
        NextChar;
      end;
    '.':
      begin
        _type := TQueryTokenType.Dot;
        NextChar;
      end;
    '''':
      begin
        // Let's just build the string with concatenation, query strings are small
        FStringValue := ExtractString;
        _type := TQueryTokenType.TextLiteral;
      end;
    '-':
      begin
        NextChar;
        if IsDigit(PeekChar) then
        begin
          Dec(FPos);
          _type := ParseFromDigit;
          if not IsNumeric(_type) then
          begin
            FPos := TokenLastPos;
            NextChar;
            _type := TQueryTokenType.Negate;
          end;
        end
        else
          _type := TQueryTokenType.Negate;
      end;
    #0: _type := TQueryTokenType.EOF;
  else
    if IsDigit(C) then
      _type := ParseFromDigit
    else
    if IsIdentifierStart(C) then
    begin
      ParseIdentifier;
      if (PeekChar = '-') and TryParseGuid(TokenLastPos) then
        _type := TQueryTokenType.GuidLiteral
      else
        _type := TQueryTokenType.Identifier;
    end
    else
      raise EQueryInvalidCharError.Create(Self, C);
  end;

  TokenText := Copy(FExpr, TokenLastPos, FPos - TokenLastPos);

  // if token is identifier, check for special identifiers
  if _type = TQueryTokenType.Identifier then
  begin
    FIdValue := TokenText;
    if (TokenText = 'true') or (TokenText = 'True') then
    begin
      _type := TQueryTokenType.BooleanLiteral;
      FBooleanValue := true;
    end
    else
    if (TokenText = 'false') or (TokenText = 'False') then
    begin
      _type := TQueryTokenType.BooleanLiteral;
      FBooleanValue := false;
    end
    else
    if (TokenText = 'null') or (TokenText = 'Null') then
    begin
      _type := TQueryTokenType.NullLiteral;
    end
    else
    if PeekChar = '''' then
    begin
      // Check for special literals which are an identifier followed by quoted string
      // for example, binary'012345' or X'asdfadfa'
//      IdLower := LowerCase(Id);
//      if (IdLower = 'datetime') then
//        ParseDateTime
//      else
//      if (IdLower = 'guid') then
//        ParseGuid
//      else
//      if (IdLower = 'time') then
//        ParseTime
//      else
//      if (IdLower = 'datetimeoffset') then
//        ParseDateTimeOffset
//      else
      if (TokenText = 'X') or SameText(TokenText, 'binary') then
        ParseBinary
    end
  end;

  FIntTokenType := _type;
  Result := FIntTokenType;
end;

procedure TQueryTokenizer.ParseBinary;
begin
  raise EBaseQuerySyntaxError.Create(Self, 'Binary literal not supported');
end;

procedure TQueryTokenizer.ParseIdentifier;
begin
  repeat
    NextChar;
  until not IsIdentifierPart(PeekChar);
end;

function TQueryTokenizer.ParseLiteral(TokenPos: integer): string;
var
  C: Char;
begin
  repeat
     NextChar;
     C := PeekChar;
  until (C = #0) or (C = ',') or (C = ')') or (C = ' ');
  if C = #0 then
    NextChar;
  Result := Copy(FExpr, TokenPos, FPos - TokenPos);
end;

function TQueryTokenizer.ParseFromDigit: TQueryTokenType;
var
  TokenPos: integer;
  S: string;
  Trailing: integer;
begin
  Trailing := 0;
  TokenPos := FPos;
  Result := TQueryTokenType.IntegerLiteral;
  repeat
    NextChar;
  until not IsDigit(PeekChar);

  if PeekChar = '-' then
  begin
    if TryParseDateTime(TokenPos) then
      Exit(TQueryTokenType.DateTimeLiteral)
    else
    if TryParseGuid(TokenPos) then
      Exit(TQueryTokenType.GuidLiteral);
  end;

  if IsLetter(PeekChar) and TryParseGuid(TokenPos) then
    Exit(TQueryTokenType.GuidLiteral);

  if PeekChar = '.' then
  begin
    Result := TQueryTokenType.DoubleLiteral;
    NextChar;
    ValidateDigit;
    repeat
      NextChar;
    until not IsDigit(PeekChar);
  end;

  if (PeekChar = 'E') or (PeekChar = 'e') then
  begin
    Result := TQueryTokenType.DoubleLiteral;
    NextChar;
    if (PeekChar = '-') or (PeekChar = '+') then
      NextChar;
    ValidateDigit;
    repeat
      NextChar;
    until not IsDigit(PeekChar);
  end;

  case PeekChar of
    'F', 'f', 'M', 'm', 'D', 'd':
      begin
        Result := TQueryTokenType.DoubleLiteral;
        Trailing := 1;
        NextChar;
      end;
    'L', 'l':
      begin
        if Result = TQueryTokenType.DoubleLiteral then
          raise EQueryInvalidNumberError.Create(Self);
        Result := TQueryTokenType.Int64Literal;
        Trailing := 1;
        NextChar;
      end;
  end;

  S := Copy(FExpr, TokenPos, FPos - TokenPos - Trailing);

  case Result of
    TQueryTokenType.IntegerLiteral:
      if not TryStrToInt(S, FIntegerValue) then
      begin
        if TryStrToInt64(S, FInt64Value) then
          Result := TQueryTokenType.Int64Literal
        else
          raise EQueryInvalidNumberError.Create(Self);
      end;
    TQueryTokenType.Int64Literal:
      if not TryStrToInt64(S, FInt64Value) then
        raise EQueryInvalidNumberError.Create(Self);
  else
    if not TryStrToFloat(S, FDoubleValue, FFormatSettings) then
      raise EQueryInvalidNumberError.Create(Self);
  end;
end;

function TQueryTokenizer.PeekChar: Char;
begin
  if not EOF then
    Result := FExpr[FPos]
  else
    Result := #0;
end;

function TQueryTokenizer.SkipWhitespace: Char;
begin
  while IsWhiteSpace(PeekChar) do
    MoveNext;
  Result := PeekChar;
end;

function TQueryTokenizer.TryParseDateTime(TokenPos: integer): boolean;
var
  InitialIndex: integer;
begin
  InitialIndex := FPos;
  Result := TSparkleUtils.TryISOToDateTime(ParseLiteral(TokenPos), FDateTimeValue);
  if not Result then
    FPos := InitialIndex;
end;

function TQueryTokenizer.TryParseGuid(TokenPos: integer): boolean;
var
  InitialIndex: integer;
begin
  InitialIndex := FPos;
  Result := TSparkleUtils.TryStringToGuid(ParseLiteral(TokenPos), FGuidValue);
  if not Result then
    FPos := InitialIndex;
end;

procedure TQueryTokenizer.ValidateDigit;
begin
  if not IsDigit(PeekChar) then
    raise EQueryInvalidNumberError.Create(Self);
end;

end.
