From 4aa7653f76f2223c41debdfdddf6a420d3f0db88 Mon Sep 17 00:00:00 2001
From: Jed Barber <jjbarber@y7mail.com>
Date: Sun, 19 Apr 2020 22:57:34 +1000
Subject: Refactoring of Lexer Scan functions
---
src/packrat-lexer.adb | 142 ++++++++++++++++++++------------------------------
src/packrat-lexer.ads | 21 +++++---
2 files changed, 71 insertions(+), 92 deletions(-)
diff --git a/src/packrat-lexer.adb b/src/packrat-lexer.adb
index faf8f71..256341e 100644
--- a/src/packrat-lexer.adb
+++ b/src/packrat-lexer.adb
@@ -57,7 +57,8 @@ package body Packrat.Lexer is
end if;
else
Context.Status := Current_Result.Status;
- Context.Pass_Forward.Replace_Element (Input (Context.Position .. Current_Result.Finish));
+ Context.Pass_Forward.Replace_Element
+ (Input (Context.Position .. Current_Result.Finish));
Context.Empty_Labels.Clear;
end if;
@@ -98,7 +99,8 @@ package body Packrat.Lexer is
end if;
else
Context.Status := Current_Result.Status;
- Context.Pass_Forward.Replace_Element (Input (Context.Position .. Current_Result.Finish));
+ Context.Pass_Forward.Replace_Element
+ (Input (Context.Position .. Current_Result.Finish));
Context.Empty_Labels.Clear;
end if;
@@ -114,7 +116,7 @@ package body Packrat.Lexer is
(Details : in out Lexer_Context;
Number_Comp : in Ada.Containers.Count_Type) is
begin
- Details.Pass_Forward := Input_Holders.Empty_Holder;
+ Details.Pass_Forward.Clear;
Details.Empty_Labels.Clear;
Details.Error_Labels.Clear;
@@ -178,30 +180,21 @@ package body Packrat.Lexer is
return Gen_Tokens.Token_Array
is
Real_Input : Input_Holders.Holder;
- Raise_Error : Boolean;
begin
if not Context.Pass_Forward.Is_Empty then
- Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Input);
+ Real_Input.Replace_Element (Context.Pass_Forward.Element & Input);
else
- Real_Input := Input_Holders.To_Holder (Input);
+ Real_Input.Replace_Element (Input);
end if;
Tidy_Context (Context, Components'Length);
Context.Result_So_Far.Clear;
Context.Allow_Incomplete := Input'Length > 0;
- while Context.Status = Success and Context.Position <= Real_Input.Constant_Reference.Element'Length loop
- Raise_Error := True;
- for C of Components loop
- if C (Real_Input.Element, Context) = Component_Success then
- Raise_Error := False;
- exit;
- end if;
- end loop;
- if Raise_Error then
- Raise_Lexer_Error (Context.Error_Labels, Context.Position);
- end if;
+ while Context.Status = Success and Context.Position <= Real_Input.Element'Length loop
+ Internal_Scan_Core (Real_Input.Element, Context, Components);
end loop;
+
return Token_Vector_To_Array (Context.Result_So_Far);
end Scan;
@@ -212,30 +205,21 @@ package body Packrat.Lexer is
return Gen_Tokens.Token_Array
is
Real_Input : Input_Holders.Holder;
- Raise_Error : Boolean;
begin
if not Context.Pass_Forward.Is_Empty then
- Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Input);
+ Real_Input.Replace_Element (Context.Pass_Forward.Element & Input);
else
- Real_Input := Input_Holders.To_Holder (Input);
+ Real_Input.Replace_Element (Input);
end if;
Tidy_Context (Context, Components'Length);
Context.Result_So_Far.Clear;
Context.Allow_Incomplete := False;
- while Context.Status = Success and Context.Position <= Real_Input.Constant_Reference.Element'Length loop
- Raise_Error := True;
- for C of Components loop
- if C (Real_Input.Element, Context) = Component_Success then
- Raise_Error := False;
- exit;
- end if;
- end loop;
- if Raise_Error then
- Raise_Lexer_Error (Context.Error_Labels, Context.Position);
- end if;
+ while Context.Status = Success and Context.Position <= Real_Input.Element'Length loop
+ Internal_Scan_Core (Real_Input.Element, Context, Components);
end loop;
+
return Token_Vector_To_Array (Context.Result_So_Far);
end Scan_Only;
@@ -246,30 +230,21 @@ package body Packrat.Lexer is
return Gen_Tokens.Token_Array
is
Real_Input : Input_Holders.Holder;
- Empty_Input, Raise_Error : Boolean;
+ Empty_Input : Boolean;
begin
Context.Result_So_Far.Clear;
loop
- Real_Input := Input_Holders.To_Holder (Input.all);
- Empty_Input := Real_Input.Constant_Reference.Element'Length = 0;
+ Real_Input.Replace_Element (Input.all);
+ Empty_Input := Real_Input.Element'Length = 0;
if not Context.Pass_Forward.Is_Empty then
- Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Real_Input.Element);
+ Real_Input.Replace_Element (Context.Pass_Forward.Element & Real_Input.Element);
end if;
Tidy_Context (Context, Components'Length);
Context.Allow_Incomplete := not Empty_Input;
- while Context.Status = Success and Context.Position <= Real_Input.Constant_Reference.Element'Length loop
- Raise_Error := True;
- for C of Components loop
- if C (Real_Input.Element, Context) = Component_Success then
- Raise_Error := False;
- exit;
- end if;
- end loop;
- if Raise_Error then
- Raise_Lexer_Error (Context.Error_Labels, Context.Position);
- end if;
+ while Context.Status = Success and Context.Position <= Real_Input.Element'Length loop
+ Internal_Scan_Core (Real_Input.Element, Context, Components);
end loop;
if Empty_Input then
@@ -286,12 +261,11 @@ package body Packrat.Lexer is
Output : out Gen_Tokens.Token_Array)
is
Real_Input : Input_Holders.Holder;
- Raise_Error : Boolean;
begin
if not Context.Pass_Forward.Is_Empty then
- Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Input);
+ Real_Input.Replace_Element (Context.Pass_Forward.Element & Input);
else
- Real_Input := Input_Holders.To_Holder (Input);
+ Real_Input.Replace_Element (Input);
end if;
Tidy_Context (Context, Components'Length);
@@ -300,26 +274,16 @@ package body Packrat.Lexer is
while Context.Status = Success and then
Integer (Context.Result_So_Far.Length) < Output'Length and then
- Context.Position <= Real_Input.Constant_Reference.Element'Length and then
- Real_Input.Constant_Reference.Element (Context.Position) /= Pad_In
+ Context.Position <= Real_Input.Element'Length and then
+ Real_Input.Element (Context.Position) /= Pad_In
loop
- Raise_Error := True;
- for C of Components loop
- if C (Real_Input.Element, Context) = Component_Success then
- Raise_Error := False;
- exit;
- end if;
- end loop;
- if Raise_Error then
- Raise_Lexer_Error (Context.Error_Labels, Context.Position);
- end if;
+ Internal_Scan_Core (Real_Input.Element, Context, Components);
end loop;
- -- suspect this is wrong, test more
- if Integer (Context.Result_So_Far.Length) >= Output'Length then
- Context.Pass_Forward.Replace_Element (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
+ if Integer (Context.Result_So_Far.Length) = Output'Length then
+ Context.Pass_Forward.Replace_Element
+ (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
end if;
-
Token_Vector_To_Array (Context.Result_So_Far, Pad_Out, Output);
end Scan_Set;
@@ -330,13 +294,13 @@ package body Packrat.Lexer is
Output : out Gen_Tokens.Token_Array)
is
Real_Input : Input_Holders.Holder;
- Empty_Input, Raise_Error : Boolean;
+ Empty_Input : Boolean;
begin
Context.Result_So_Far.Clear;
loop
- Real_Input := Input_Holders.To_Holder (Input.all);
- Empty_Input := Real_Input.Constant_Reference.Element'Length = 0 or
- Real_Input.Constant_Reference.Element (Real_Input.Constant_Reference.Element'First) = Pad_In;
+ Real_Input.Replace_Element (Input.all);
+ Empty_Input := Real_Input.Element'Length = 0 or
+ Real_Input.Element (Real_Input.Element'First) = Pad_In;
if not Context.Pass_Forward.Is_Empty then
Real_Input.Replace_Element (Context.Pass_Forward.Element & Real_Input.Element);
end if;
@@ -346,28 +310,19 @@ package body Packrat.Lexer is
while Context.Status = Success and then
Integer (Context.Result_So_Far.Length) < Output'Length and then
- Context.Position <= Real_Input.Constant_Reference.Element'Length and then
- Real_Input.Constant_Reference.Element (Context.Position) /= Pad_In
+ Context.Position <= Real_Input.Element'Length and then
+ Real_Input.Element (Context.Position) /= Pad_In
loop
- Raise_Error := True;
- for C of Components loop
- if C (Real_Input.Element, Context) = Component_Success then
- Raise_Error := False;
- exit;
- end if;
- end loop;
- if Raise_Error then
- Raise_Lexer_Error (Context.Error_Labels, Context.Position);
- end if;
+ Internal_Scan_Core (Real_Input.Element, Context, Components);
end loop;
if Empty_Input then
exit;
end if;
- -- suspect this is wrong, test more
- if Integer (Context.Result_So_Far.Length) >= Output'Length then
- Context.Pass_Forward.Replace_Element (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
+ if Integer (Context.Result_So_Far.Length) = Output'Length then
+ Context.Pass_Forward.Replace_Element
+ (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
exit;
end if;
end loop;
@@ -375,7 +330,24 @@ package body Packrat.Lexer is
end Scan_Set_With;
- -- factor out the internal scan loop to an internal function/procedure here
+ procedure Internal_Scan_Core
+ (Input : in Element_Array;
+ Context : in out Lexer_Context;
+ Components : in Component_Array)
+ is
+ Raise_Error : Boolean;
+ begin
+ Raise_Error := True;
+ for C of Components loop
+ if C (Input, Context) = Component_Success then
+ Raise_Error := False;
+ exit;
+ end if;
+ end loop;
+ if Raise_Error then
+ Raise_Lexer_Error (Context.Error_Labels, Context.Position);
+ end if;
+ end Internal_Scan_Core;
diff --git a/src/packrat-lexer.ads b/src/packrat-lexer.ads
index 693064d..9973cdd 100644
--- a/src/packrat-lexer.ads
+++ b/src/packrat-lexer.ads
@@ -293,20 +293,19 @@ private
- type Lexer_Context is new Ada.Finalization.Controlled with record
+ type Lexer_Context is record
Result_So_Far : Token_Vectors.Vector;
- Position : Positive;
- Offset : Natural;
- Status : Result_Status;
+ Position : Positive := 1;
+ Offset : Natural := 0;
+ Status : Result_Status := Success;
Pass_Forward : Input_Holders.Holder;
Empty_Labels : Label_Sets.Set;
Error_Labels : Label_Vectors.Vector;
- Allow_Incomplete : Boolean;
+ Allow_Incomplete : Boolean := True;
end record;
Empty_Context : constant Lexer_Context :=
- (Ada.Finalization.Controlled with
- Result_So_Far => Token_Vectors.Empty_Vector,
+ (Result_So_Far => Token_Vectors.Empty_Vector,
Position => 1,
Offset => 0,
Status => Success,
@@ -316,6 +315,14 @@ private
Allow_Incomplete => True);
+
+
+ procedure Internal_Scan_Core
+ (Input : in Element_Array;
+ Context : in out Lexer_Context;
+ Components : in Component_Array);
+
+
end Packrat.Lexer;
--
cgit