Adding token collection to lex.py for REPL project

asteroid/lex.py

@@ -126,6 +126,49 @@ def init_token_values():
 def token_lookup(type):
     return token_values[type]
 
+#This returns all of the token identifiers currently defined.
+#The pool of identifiers includes both user-defined data
+#and keywords defined by the language. The list of identifiers
+#does not contain duplicates and is sorted alphanumerically.
+def get_indentifiers() -> list[str]:
+    #A set is used to only have unique elements
+    ids = set()
+    for name in keywords:
+        ids.add(name)
+
+    for table in state.symbol_table.scoped_symtab:
+        for name in table:
+            ids.add(name)
+
+    #We want a list so we can index into it
+    ids = list(ids)
+    ids.sort()
+
+    return ids
+
+#This returns all of the token identifiers currently defined which
+#are prefixed by the input string (i.e. it begins with the string).
+#The pool of identifiers includes both user-defined data
+#and keywords defined by the language. The list of identifiers
+#does not contain duplicates and is sorted alphanumerically.
+def get_indentifiers_by_prefix(prefix: str) -> list[str]:
+    #A set is used to only have unique elements
+    ids = set()
+    for name in keywords:
+        if name.startswith(prefix):
+            ids.add(name)
+
+    for table in state.symbol_table.scoped_symtab:
+        for name in table:
+            if name.startswith(prefix):
+                ids.add(name)
+
+    #We want a list so we can index into it
+    ids = list(ids)
+    ids.sort()
+
+    return ids
+
 class Token:
     def __init__(self,type,value,module,lineno):
         self.type = type

asteroid/test-suites/python-tests/README.md

@@ -0,0 +1 @@
+The tests in this suite are not run when run-tests.py is run. Currently, each file must be run individually.

asteroid/test-suites/python-tests/repl_test.py

@@ -0,0 +1,48 @@
+import sys
+import os
+
+#Code from run-tests.py
+file_path = os.path.dirname(os.path.abspath( __file__ ))
+os.chdir(file_path)
+(parent_dir,_) = os.path.split(file_path)
+#We need the granparent dir, since we're 2 layers down from asteroid
+(grandparent_dir,_) = os.path.split(parent_dir)
+sys.path.append(grandparent_dir)
+
+
+from asteroid.interp import interp
+import lex
+from state import state
+
+def test_get_identifiers():
+    print(lex.get_indentifiers())
+    interp("load io.", "test")
+    print(lex.get_indentifiers())
+    interp("let aaa = 1.", "test")
+    interp("let aab = 2.", "test", initialize_state=False)
+    interp("let abc = 3.", "test", initialize_state=False)
+    interp("let bc = 4.", "test", initialize_state=False)
+    interp("let baa = 5.", "test", initialize_state=False)
+    print(lex.get_indentifiers())
+
+def test_get_identifiers_by_prefix():
+    print(lex.get_indentifiers())
+    interp("load io.", "test")
+    print(lex.get_indentifiers())
+    interp("let aaa = 1.", "test")
+    interp("let aab = 2.", "test", initialize_state=False)
+    interp("let abc = 3.", "test", initialize_state=False)
+    interp("let bc = 4.", "test", initialize_state=False)
+    interp("let baa = 5.", "test", initialize_state=False)
+    print(lex.get_indentifiers())
+    print(lex.get_indentifiers_by_prefix("a"))
+    print(lex.get_indentifiers_by_prefix("aa"))
+    print(lex.get_indentifiers_by_prefix("__"))
+
+
+if __name__ == "__main__":
+    print("Testing get_identifiers:")
+    test_get_identifiers()
+    print("\n\n\n")
+    print("Testing get_identifiers_by_prefix")
+    test_get_identifiers_by_prefix()