From da389927ddf9240bbba10b819eb782e80a5d6bf7 Mon Sep 17 00:00:00 2001 From: Jed Barber Date: Tue, 19 Jan 2021 02:25:44 +1100 Subject: Basic HTML documentation added --- doc/lexer_comb.html | 347 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 347 insertions(+) create mode 100644 doc/lexer_comb.html (limited to 'doc/lexer_comb.html') diff --git a/doc/lexer_comb.html b/doc/lexer_comb.html new file mode 100644 index 0000000..eef5c04 --- /dev/null +++ b/doc/lexer_comb.html @@ -0,0 +1,347 @@ + + + + + + + Lexer Components - Packrat Docs + + + + + + +

Lexer Components

+ + Return to Contents + + +

Lexers are a much simplified version of the parsers. They operate like one giant + Choice combinator, the other combinators that you can combine with are more + limited, all combinators operate in the more usual greedy consume-as-much-as-possible + fashion, and the output is a sequence of tokens instead of a parse graph.
+
+ Think of it like grouping items of input together and labelling the groups, instead of + trying to create any complex parses.
+
+ Usage of lexers is optional.

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 
+generic
+    Components : in Component_Array;
+package Scan_Parts is
+
+    function Scan
+           (Input : in Traits.Element_Array)
+        return Traits.Tokens.Token_Array;
+
+    procedure Reset;
+
+end Scan_Parts;
+
The scanner version that allows for piecewise scanning. The Scan function can be +called repeatedly with new input until it is supplied with an empty array, at which point it +is assumed that the end of input has been reached.
+
+The Reset procedure reverts the scanner to a clean state the same as if it was just +declared, with no partial results stored internally.
+generic
+    Components : in Component_Array;
+package Scan_Once is
+
+    function Scan
+           (Input : in Traits.Element_Array)
+        return Traits.Tokens.Token_Array;
+
+    procedure Reset;
+
+end Scan_Once;
+
This scanner version assumes that the input supplied is the only input available.
+
+The Reset procedure reverts the scanner to a clean state the same as if it was just +declared.
+generic
+    Components : in Component_Array;
+package Scan_With is
+
+    function Scan
+           (Input : in With_Input)
+        return Traits.Tokens.Token_Array;
+
+    procedure Reset;
+
+end Scan_With;
+
This scanner uses the supplied function to retrieve input to be lexed. Like the Scan_Parts +scanner it will continue to call the function until an empty array is returned, at which point it is +assumed that the end of input has been reached.
+
+The Reset procedure reverts the scanner to a clean state the same as if it was just +declared.
+generic
+    Components : in Component_Array;
+    Pad_In     : in Traits.Element_Type;
+    Pad_Out    : in Traits.Tokens.Token_Type;
+package Scan_Set is
+
+    procedure Scan
+           (Input  : in     Traits.Element_Array;
+            Output :    out Traits.Tokens.Token_Array);
+
+    procedure Reset;
+
+end Scan_Set;
+
This scanner allows the use of constant length input and output arrays. Scanning is done piecewise +until the input array begins with the supplied Pad_In character, upon which it is assumed that +the end of input has been reached. Any unused space in the output array is padded with the Pad_Out +character.
+
+The Reset procedure reverts the scanner to a clean state the same as if it was just declared.
+
+Note that the input and output array lengths do not have to remain the same between successive calls to +Scan.
+generic
+    Components : in Component_Array;
+    Pad_In     : in Traits.Element_Type;
+    Pad_Out    : in Traits.Tokens.Token_Type;
+package Scan_Set_With is
+
+    procedure Scan
+           (Input  : in     With_Input;
+            Output :    out Traits.Tokens.Token_Array);
+
+    procedure Reset;
+
+end Scan_Set_With;
+
A combination of Scan_With and Scan_Set, this scanner uses the supplied function +to retrieve input to be lexed, but uses padding characters in the same manner as Scan_Set.
+
+The Reset procedure reverts the scanner to a clean state the same as if it was just declared.
+generic
+    Label : in Traits.Label_Enum;
+    with function Combo
+           (Input : in Traits.Element_Array;
+            Start : in Positive)
+        return Combinator_Result;
+function Stamp
+       (Input   : in     Traits.Element_Array;
+        Context : in out Lexer_Context)
+    return Component_Result;
+
Runs the supplied combinator, and if successful converts the result into a token with the supplied +label that then gets added to the output. Scanning then proceeds from just after the input covered by +the token.
+generic
+    Label : in Traits.Label_Enum;
+    with function Combo
+           (Input : in Traits.Element_Array;
+            Start : in Positive)
+        return Combinator_Result;
+function Discard
+       (Input   : in     Traits.Element_Array;
+        Context : in out Lexer_Context)
+    return Component_Result;
+
Does the same thing as Stamp, however rather than adding the token to the output it instead +discards it. Scanning still proceeds from just after the input covered by the token.
+
+Even though the result is ultimately discarded it is nonetheless important that a label is used here so +that informative error messages can be generated.
+generic
+    Params : in Combinator_Array;
+function Sequence
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes the supplied combinator Params in order. If they are all successful then the joined +successful result is returned, otherwise the combinator fails.
+generic
+    with function Param
+           (Input : in Traits.Element_Array;
+            Start : in Positive)
+        return Combinator_Result;
+    Number : in Positive;
+function Count
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes the supplied combinator Param the supplied Number of times one after another. +If it is all successful then the joined successful result is returned, otherwise the combinator fails.
+generic
+    with function Param
+           (Input : in Traits.Element_Array;
+            Start : in Positive)
+        return Combinator_Result;
+    Minimum : in Natural := 0;
+function Many
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes the supplied combinator Param as many times as it can. Succeeds if it manages to do +so at least Minimum number of times.
+generic
+    with function Param
+           (Input : in Traits.Element_Array;
+            Start : in Positive)
+        return Combinator_Result;
+    with function Test
+           (Item : in Traits.Element_Type)
+        return Boolean;
+    Minimum : in Natural := 0;
+function Many_Until
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes the supplied combinator Param over and over until Test succeeds on the next +available item in the input. Succeeds if it manages to lex Param at least Minimum times. +Fails if Param fails before Test succeeds on the next available item of input.
+generic
+    with function Test
+           (Item : in Traits.Element_Type)
+        return Boolean;
+function Satisfy
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes one item of input if the supplied Test function succeeds on that item, otherwise fails.
+generic
+    with function Test
+           (Item : in Traits.Element_Type)
+        return Boolean;
+    with function Change
+           (From : in Traits.Element_Type)
+        return Traits.Element_Type;
+function Satisfy_With
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
First uses the supplied Change function on the first item of input, then acts as in Satisfy +on the changed item. Upon success the item lexed is the original item.
+generic
+    Item : in Traits.Element_Type;
+function Match
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes one item of input if it matches Item.
+generic
+    Item : in Traits.Element_Type;
+    with function Change
+           (From : in Traits.Element_Type)
+        return Traits.Element_Type;
+function Match_With
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes one item of input if, after applying the supplied Change function to it, it +matches Item.
+generic
+    Items : in Traits.Element_Array;
+function Multimatch
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes the next Items'Length items of input if they match Items.
+generic
+    Number : in Positive := 1;
+function Take
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes the next Number items of input.
+generic
+    with function Test
+           (Item : in Traits.Element_Type)
+        return Boolean;
+function Take_While
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes successive items of input as long as they satisfy the supplied Test function.
+generic
+    with function Test
+           (Item : in Traits.Element_Type)
+        return Boolean;
+function Take_Until
+       (Input : in Traits.Element_Array;
+        Start : in Positive)
+    return Combinator_Result;
+
Lexes items of input until the next available item satisfies the supplied Test function.
+ + + + + -- cgit