Refactoring of Lexer Scan functions

2 files changed, 71 insertions, 92 deletions

diff --git a/src/packrat-lexer.adb b/src/packrat-lexer.adb
index faf8f71..256341e 100644
--- a/src/packrat-lexer.adb
+++ b/src/packrat-lexer.adb
@@ -57,7 +57,8 @@ package body Packrat.Lexer is
             end if;
         else
             Context.Status := Current_Result.Status;
-            Context.Pass_Forward.Replace_Element (Input (Context.Position .. Current_Result.Finish));
+            Context.Pass_Forward.Replace_Element
+                (Input (Context.Position .. Current_Result.Finish));
             Context.Empty_Labels.Clear;
         end if;
 
@@ -98,7 +99,8 @@ package body Packrat.Lexer is
             end if;
         else
             Context.Status := Current_Result.Status;
-            Context.Pass_Forward.Replace_Element (Input (Context.Position .. Current_Result.Finish));
+            Context.Pass_Forward.Replace_Element
+                (Input (Context.Position .. Current_Result.Finish));
             Context.Empty_Labels.Clear;
         end if;
 
@@ -114,7 +116,7 @@ package body Packrat.Lexer is
            (Details     : in out Lexer_Context;
             Number_Comp : in     Ada.Containers.Count_Type) is
     begin
-        Details.Pass_Forward := Input_Holders.Empty_Holder;
+        Details.Pass_Forward.Clear;
 
         Details.Empty_Labels.Clear;
         Details.Error_Labels.Clear;
@@ -178,30 +180,21 @@ package body Packrat.Lexer is
         return Gen_Tokens.Token_Array
     is
         Real_Input : Input_Holders.Holder;
-        Raise_Error : Boolean;
     begin
         if not Context.Pass_Forward.Is_Empty then
-            Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Input);
+            Real_Input.Replace_Element (Context.Pass_Forward.Element & Input);
         else
-            Real_Input := Input_Holders.To_Holder (Input);
+            Real_Input.Replace_Element (Input);
         end if;
 
         Tidy_Context (Context, Components'Length);
         Context.Result_So_Far.Clear;
         Context.Allow_Incomplete := Input'Length > 0;
 
-        while Context.Status = Success and Context.Position <= Real_Input.Constant_Reference.Element'Length loop
-            Raise_Error := True;
-            for C of Components loop
-                if C (Real_Input.Element, Context) = Component_Success then
-                    Raise_Error := False;
-                    exit;
-                end if;
-            end loop;
-            if Raise_Error then
-                Raise_Lexer_Error (Context.Error_Labels, Context.Position);
-            end if;
+        while Context.Status = Success and Context.Position <= Real_Input.Element'Length loop
+            Internal_Scan_Core (Real_Input.Element, Context, Components);
         end loop;
+
         return Token_Vector_To_Array (Context.Result_So_Far);
     end Scan;
 
@@ -212,30 +205,21 @@ package body Packrat.Lexer is
         return Gen_Tokens.Token_Array
     is
         Real_Input : Input_Holders.Holder;
-        Raise_Error : Boolean;
     begin
         if not Context.Pass_Forward.Is_Empty then
-            Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Input);
+            Real_Input.Replace_Element (Context.Pass_Forward.Element & Input);
         else
-            Real_Input := Input_Holders.To_Holder (Input);
+            Real_Input.Replace_Element (Input);
         end if;
 
         Tidy_Context (Context, Components'Length);
         Context.Result_So_Far.Clear;
         Context.Allow_Incomplete := False;
 
-        while Context.Status = Success and Context.Position <= Real_Input.Constant_Reference.Element'Length loop
-            Raise_Error := True;
-            for C of Components loop
-                if C (Real_Input.Element, Context) = Component_Success then
-                    Raise_Error := False;
-                    exit;
-                end if;
-            end loop;
-            if Raise_Error then
-                Raise_Lexer_Error (Context.Error_Labels, Context.Position);
-            end if;
+        while Context.Status = Success and Context.Position <= Real_Input.Element'Length loop
+            Internal_Scan_Core (Real_Input.Element, Context, Components);
         end loop;
+
         return Token_Vector_To_Array (Context.Result_So_Far);
     end Scan_Only;
 
@@ -246,30 +230,21 @@ package body Packrat.Lexer is
         return Gen_Tokens.Token_Array
     is
         Real_Input : Input_Holders.Holder;
-        Empty_Input, Raise_Error : Boolean;
+        Empty_Input : Boolean;
     begin
         Context.Result_So_Far.Clear;
         loop
-            Real_Input := Input_Holders.To_Holder (Input.all);
-            Empty_Input := Real_Input.Constant_Reference.Element'Length = 0;
+            Real_Input.Replace_Element (Input.all);
+            Empty_Input := Real_Input.Element'Length = 0;
             if not Context.Pass_Forward.Is_Empty then
-                Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Real_Input.Element);
+                Real_Input.Replace_Element (Context.Pass_Forward.Element & Real_Input.Element);
             end if;
 
             Tidy_Context (Context, Components'Length);
             Context.Allow_Incomplete := not Empty_Input;
 
-            while Context.Status = Success and Context.Position <= Real_Input.Constant_Reference.Element'Length loop
-                Raise_Error := True;
-                for C of Components loop
-                    if C (Real_Input.Element, Context) = Component_Success then
-                        Raise_Error := False;
-                        exit;
-                    end if;
-                end loop;
-                if Raise_Error then
-                    Raise_Lexer_Error (Context.Error_Labels, Context.Position);
-                end if;
+            while Context.Status = Success and Context.Position <= Real_Input.Element'Length loop
+                Internal_Scan_Core (Real_Input.Element, Context, Components);
             end loop;
 
             if Empty_Input then
@@ -286,12 +261,11 @@ package body Packrat.Lexer is
             Output  :    out Gen_Tokens.Token_Array)
     is
         Real_Input : Input_Holders.Holder;
-        Raise_Error : Boolean;
     begin
         if not Context.Pass_Forward.Is_Empty then
-            Real_Input := Input_Holders.To_Holder (Context.Pass_Forward.Element & Input);
+            Real_Input.Replace_Element (Context.Pass_Forward.Element & Input);
         else
-            Real_Input := Input_Holders.To_Holder (Input);
+            Real_Input.Replace_Element (Input);
         end if;
 
         Tidy_Context (Context, Components'Length);
@@ -300,26 +274,16 @@ package body Packrat.Lexer is
 
         while Context.Status = Success and then
             Integer (Context.Result_So_Far.Length) < Output'Length and then
-            Context.Position <= Real_Input.Constant_Reference.Element'Length and then
-            Real_Input.Constant_Reference.Element (Context.Position) /= Pad_In
+            Context.Position <= Real_Input.Element'Length and then
+            Real_Input.Element (Context.Position) /= Pad_In
         loop
-            Raise_Error := True;
-            for C of Components loop
-                if C (Real_Input.Element, Context) = Component_Success then
-                    Raise_Error := False;
-                    exit;
-                end if;
-            end loop;
-            if Raise_Error then
-                Raise_Lexer_Error (Context.Error_Labels, Context.Position);
-            end if;
+            Internal_Scan_Core (Real_Input.Element, Context, Components);
         end loop;
 
-        --  suspect this is wrong, test more
-        if Integer (Context.Result_So_Far.Length) >= Output'Length then
-            Context.Pass_Forward.Replace_Element (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
+        if Integer (Context.Result_So_Far.Length) = Output'Length then
+            Context.Pass_Forward.Replace_Element
+                (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
         end if;
-
         Token_Vector_To_Array (Context.Result_So_Far, Pad_Out, Output);
     end Scan_Set;
 
@@ -330,13 +294,13 @@ package body Packrat.Lexer is
             Output  :    out Gen_Tokens.Token_Array)
     is
         Real_Input : Input_Holders.Holder;
-        Empty_Input, Raise_Error : Boolean;
+        Empty_Input : Boolean;
     begin
         Context.Result_So_Far.Clear;
         loop
-            Real_Input := Input_Holders.To_Holder (Input.all);
-            Empty_Input := Real_Input.Constant_Reference.Element'Length = 0 or
-                Real_Input.Constant_Reference.Element (Real_Input.Constant_Reference.Element'First) = Pad_In;
+            Real_Input.Replace_Element (Input.all);
+            Empty_Input := Real_Input.Element'Length = 0 or
+                Real_Input.Element (Real_Input.Element'First) = Pad_In;
             if not Context.Pass_Forward.Is_Empty then
                 Real_Input.Replace_Element (Context.Pass_Forward.Element & Real_Input.Element);
             end if;
@@ -346,28 +310,19 @@ package body Packrat.Lexer is
 
             while Context.Status = Success and then
                 Integer (Context.Result_So_Far.Length) < Output'Length and then
-                Context.Position <= Real_Input.Constant_Reference.Element'Length and then
-                Real_Input.Constant_Reference.Element (Context.Position) /= Pad_In
+                Context.Position <= Real_Input.Element'Length and then
+                Real_Input.Element (Context.Position) /= Pad_In
             loop
-                Raise_Error := True;
-                for C of Components loop
-                    if C (Real_Input.Element, Context) = Component_Success then
-                        Raise_Error := False;
-                        exit;
-                    end if;
-                end loop;
-                if Raise_Error then
-                    Raise_Lexer_Error (Context.Error_Labels, Context.Position);
-                end if;
+                Internal_Scan_Core (Real_Input.Element, Context, Components);
             end loop;
 
             if Empty_Input then
                 exit;
             end if;
 
-            --  suspect this is wrong, test more
-            if Integer (Context.Result_So_Far.Length) >= Output'Length then
-                Context.Pass_Forward.Replace_Element (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
+            if Integer (Context.Result_So_Far.Length) = Output'Length then
+                Context.Pass_Forward.Replace_Element
+                    (Real_Input.Element (Context.Position .. Real_Input.Element'Last));
                 exit;
             end if;
         end loop;
@@ -375,7 +330,24 @@ package body Packrat.Lexer is
     end Scan_Set_With;
 
 
-    --  factor out the internal scan loop to an internal function/procedure here
+    procedure Internal_Scan_Core
+           (Input      : in     Element_Array;
+            Context    : in out Lexer_Context;
+            Components : in     Component_Array)
+    is
+        Raise_Error : Boolean;
+    begin
+        Raise_Error := True;
+        for C of Components loop
+            if C (Input, Context) = Component_Success then
+                Raise_Error := False;
+                exit;
+            end if;
+        end loop;
+        if Raise_Error then
+            Raise_Lexer_Error (Context.Error_Labels, Context.Position);
+        end if;
+    end Internal_Scan_Core;
 
 
 
diff --git a/src/packrat-lexer.ads b/src/packrat-lexer.ads
index 693064d..9973cdd 100644
--- a/src/packrat-lexer.ads
+++ b/src/packrat-lexer.ads
@@ -293,20 +293,19 @@ private
 
 
 
-    type Lexer_Context is new Ada.Finalization.Controlled with record
+    type Lexer_Context is record
         Result_So_Far    : Token_Vectors.Vector;
-        Position         : Positive;
-        Offset           : Natural;
-        Status           : Result_Status;
+        Position         : Positive := 1;
+        Offset           : Natural := 0;
+        Status           : Result_Status := Success;
         Pass_Forward     : Input_Holders.Holder;
         Empty_Labels     : Label_Sets.Set;
         Error_Labels     : Label_Vectors.Vector;
-        Allow_Incomplete : Boolean;
+        Allow_Incomplete : Boolean := True;
     end record;
 
     Empty_Context : constant Lexer_Context :=
-       (Ada.Finalization.Controlled with
-        Result_So_Far    => Token_Vectors.Empty_Vector,
+       (Result_So_Far    => Token_Vectors.Empty_Vector,
         Position         => 1,
         Offset           => 0,
         Status           => Success,
@@ -316,6 +315,14 @@ private
         Allow_Incomplete => True);
 
 
+
+
+    procedure Internal_Scan_Core
+           (Input      : in     Element_Array;
+            Context    : in out Lexer_Context;
+            Components : in     Component_Array);
+
+
 end Packrat.Lexer;