Added third positional variable to buildAction

core/apirequest/execute.py

@@ -2,6 +2,8 @@ import requests
 import threading
 import os
 import time
+import logging
+
 from .request import buildRequest
 
 #session = requests.Session()
@@ -48,48 +50,11 @@ def executeRequest(action, outputs, resolved):
             return response, outputs
 
         except (requests.Timeout, requests.ConnectionError) as error:
-            print(f"{type(error).__name__} during {action.name}: {error}")
+            #print(f"{type(error).__name__} during {action.name}: {error}")
+            logging.warning(f"{type(error).__name__} during {action.name}: {error}")
             session = getSession(reset=True)
 
             if attempt == 0:
-                print(f"Retrying {action.name}")
+                logging.warning(f"Retrying {action.name}")
 
     return None, outputs
-
-#    try:
-#        request, outputs = buildRequest(action, outputs, resolved)
-#
-#        response = session.request(
-#            method=request["method"],
-#            url=request["path"],
-#            json=request.get("object"),
-#            headers=request.get("headers"),
-#            data=request.get("data"),
-#            files=request.get("files"),
-#            timeout=(10, 30)
-#            )
-#
-#    except requests.ConnectionError as error:
-#        print(f"Request failed due to {error} during {action.name}")
-#
-#        session = getSession(reset=True)
-#        
-#        print(f"Retrying {action.name}")
-#
-#        response = session.request(
-#            method=request["method"],
-#            url=request["path"],
-#            json=request.get("object"),
-#            headers=request.get("headers"),
-#            data=request.get("data"),
-#            files=request.get("files"),
-#            timeout=(10, 30)
-#            )
-#
-#        return response, outputs
-#
-#    except requests.Timeout as error:
-#        print(f"Request timed out during {action.name}: {error}")
-#        session = getSession(reset=True)
-#
-#    return response, outputs

core/console.py

@@ -0,0 +1,3 @@
+from rich.console import Console
+
+console = Console(stderr=True)

core/execution/action/action.py

@@ -22,7 +22,6 @@ def buildAction (action, outputs, process):
     if response is None:
         return outputs, None
 
-    #outputs["processes"][process.name][action.name]["response"] = response
     outputs, response = buildRuntimeAttr(action, outputs, resolved, response, process)
 
     if action_log:

core/execution/monitor.py

@@ -1,3 +1,5 @@
+import logging
+
 #from core.execution.executioncontract import executeContract
 from core.execution.action import buildAction
 from core.execution.operation import buildOperation
@@ -25,9 +27,10 @@ def buildMonitor(monitor, outputs):
 
     match, filter = applyConditions(monitor, readitems, outputs)
     
-    outputs = buildOutput(response, monitor, outputs)
+    buildOutput(response, monitor, outputs)
 
     if match:
+        logging.info(filter.get("print"))
         module = filter["execute"]
 
         if module.kind == "action":

core/execution/operation/operation.py

@@ -1,14 +1,17 @@
 import logging
 import pprint
 from rich.progress import track
+from rich.console import Console
 
 from core.runtime.managefailure import manageFailure
 from core.runtime.stopsignals import checkDependency
 from core.execution.processregistry import registerProcess
-
+from core.console import console
 
 outputs = {}
 
+#console = Console(stderr=True)
+
 def buildOperation(definition, outputs):
     process_log = True
 
@@ -22,7 +25,8 @@ def buildOperation(definition, outputs):
         actions = track(
             actions,
             #description=definition.name
-            description=f"[green]{definition.name}[/green]"
+            description=f"[green]{definition.name}[/green]",
+            console=console
         )
 
     for action in actions:
@@ -57,7 +61,13 @@ def buildOperation(definition, outputs):
 
     #Print completion status after runtime processes
     if process_log:
-        logging.info(f"{definition.name} completed")
+        stopsignal_count = len(outputs["processes"][definition.name]["stopsignals"])
+        completed_count = len(outputs["processes"][definition.name]["completed"])
+
+        if completed_count == 1 and stopsignal_count == 1:
+            logging.info(f"{definition.name} bypassed")
+        else:
+            logging.info(f"{definition.name} completed")
 
     #For debugging purposes only
     process_registry_logs = False

core/execution/recursive.py

@@ -4,6 +4,7 @@ import os
 from core.runtime.stopsignals import checkDependency
 from core.execution.processregistry import registerProcess
 from rich.progress import track
+from core.console import console
 
 def buildRecursive(definition, outputs):
     process_log = True
@@ -14,6 +15,8 @@ def buildRecursive(definition, outputs):
     source = outputs.get(definition.recursiveset["source"])
 
     if source is None or source == []:
+        if recursive_log:
+            logging.debug(f"No items available in {source} for {definition.name}")
         return outputs, []
 
     outputs = registerProcess(definition, outputs)
@@ -25,15 +28,6 @@ def buildRecursive(definition, outputs):
     source = definition.recursiveset["source"]
     
     items = outputs.get(source)
-    #print(f"RECURSIVE DEBUG DURING {definition.name}")
-    #print(items)
-
-    #if items is None:
-
-    #    if recursive_log:
-    #        print(f"No items available in {source} for {definition.name}")
-    #
-    #    return outputs, None
 
     reverse = definition.recursiveset.get("reverse", False)
 
@@ -43,7 +37,8 @@ def buildRecursive(definition, outputs):
 
     for item in track(
         items,
-        description=f"[green]{definition.name}[/green]"
+        description=f"[green]{definition.name}[/green]",
+        console=console
     ):
 
     #for item in items:
@@ -78,7 +73,13 @@ def buildRecursive(definition, outputs):
             outputs["processes"][definition.name]["completed"].append(job.alias)
 
     if process_log:
-        logging.info(f"{definition.name} completed")
+        stopsignal_count = len(outputs["processes"][definition.name]["stopsignals"])
+        completed_count = len(outputs["processes"][definition.name]["completed"])
+
+        if completed_count == 1 and stopsignal_count == 1:
+            logging.info(f"{definition.name} bypassed")
+        else:
+            logging.info(f"{definition.name} completed")
 
     retain = definition.recursiveset.get("retain", True)
 

core/runtime/applyconditions.py

@@ -21,54 +21,43 @@ def applyConditions(definition, dictionary, outputs):
                     
                 if filter["operator"] == ">=":
                     if int(resolvedvalue or 0) >= int(filtervalue):
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == "<=":
                     if int(resolvedvalue or 0) <= int(filtervalue):
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == "==":
                     if resolvedvalue == filtervalue:
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == "!=":
                     if resolvedvalue != filtervalue:
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == ">":
                     if int(resolvedvalue or 0) > int(filtervalue):
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == "<":
                     if int(resolvedvalue or 0) < int(filtervalue):
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == "in":
                     if filtervalue in dictionary:
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == "is":
                     if resolvedvalue is filtervalue:
-                        print(filter["print"])
                         return True, filter
 
                 if filter["operator"] == "is not":
                     if resolvedvalue is not filtervalue:
-                        print(filter["print"])
                         return True, filter
 
             except KeyError:
                 print(f"Value not present in runtime for filtering during {definition.name}")
-                #print(filter)
                 return True, filter
-                #continue
 
             #except TypeError:
             #    print("TypeError")

core/runtime/firewall.py

@@ -1,9 +1,11 @@
 import os
+import logging
 
 from core.runtime.applyconditions import applyConditions
 
 def buildFirewall(listener, outputs):
-    firewall_log = os.environ.get("LOGS_FIREWALL", "false").lower() == "true"
+    #firewall_log = os.environ.get("LOGS_FIREWALL", "false").lower() == "true"
+    firewall_log = True
 
     status_code = 200
     outputs["firewall_allowed"] = []
@@ -13,20 +15,24 @@ def buildFirewall(listener, outputs):
 
         if match:
             status_code = filter.get("status_code", 200)
+            if firewall_log:
+                logging.info(filter.get("print"))
+                logging.info(f"Returning status_code: {status_code}")
             continue
 
         #This needs to be moved into definitions but until logic is put in place it lives here.
         if "com.loxthelion.rmass_message_id" in item.get("content", {}):
             status_code = 200
-            print("Excluding previously imported items")
+            if firewall_log:
-            print(f"Returning status_code: {status_code}")
+                logging.info("Excluding previously imported items")
+                logging.info(f"Returning status_code: {status_code}")
 
             continue
 
         outputs["firewall_allowed"].append(item)
     
-    if outputs["firewall_allowed"]:
+    if outputs["firewall_allowed"] != []:
-        return True, outputs, status_code
+        return True, status_code
 
     #Needs to be present to prevent TypeError
-    return False, outputs, status_code
+    return False, status_code

core/runtime/image.py

@@ -12,4 +12,4 @@ def buildImage(response, action, outputs):
         else:
             print("No response received"),
 
-    return outputs
+    return

core/runtime/managefailure.py

@@ -1,4 +1,4 @@
-from core.runtime.outputs import buildOutput
+#from core.runtime.outputs import buildOutput
 from core.execution.action.action import buildAction
 #from core.runtime.outputs import buildOutput
 
@@ -22,7 +22,7 @@ def manageFailure(error, action, outputs):
                 else:
                     print(f"{type(error).__name__} detected during {action.name}. Applying recovery policy.")
                     
-                    outputs, response = buildAction(repair, outputs)
+                    outputs, response = buildAction(repair, outputs, None)
 
                 return outputs
     print(f"{error}")

core/runtime/outputs.py

@@ -53,6 +53,6 @@ def buildOutput(response, action, outputs):
         #else:
         #    print("No response received"),
         
-    return outputs
+    return
 
 

core/runtime/resolver.py

@@ -93,6 +93,7 @@ def resolveItem2(action, item):
     return resolved
 
 def extractItem(action, item, resolved):
+    #UNDER RUNTIME CONTRACT ITEM IS OUTPUTS
     if hasattr(action, "extract"):
         for label, keys in action.extract["get"].items():
             #resolved[label] = {}
@@ -135,4 +136,4 @@ def extractItem(action, item, resolved):
 
             resolved[source] = deconstructed
 
-    return resolved
+    return

core/runtime/responsepolicy.py

@@ -1,17 +1,23 @@
-#from core.execution.action import buildAction
+import logging
+
 from core.apirequest import executeRequest
 
 def responsePolicy(response, action, outputs, resolved):
     if hasattr(action, "responsepolicy"):
         if not response.ok:
-            print(response)
+            logging.warning(f"{response} during {action.name}")
 
             for policy in action.responsepolicy:
                 if policy["statuscode"] == response.status_code:
-                    print(policy["print"])
+                    logging.warning(policy["print"])
 
-                    response, outputs = executeRequest(policy["execute"], outputs, resolved)
+                    newresponse, outputs = executeRequest(policy["execute"], outputs, resolved)
 
-                    return outputs
+                    if policy["response_update"]:
+                        logging.info("Updating response")
+                        print("Updating response")
+                        response = newresponse
 
-    return outputs
+                    break
+    
+    return response

core/runtime/runtimecontract.py

@@ -1,13 +1,12 @@
-from core.execution.action import action
+import logging
-from core.runtime import buildSelect, buildOutput, buildSave, loadSave, saveRow, readRow, buildLog
+
-from core.runtime.image import buildImage
-#from core.runtime.select import buildSelect3
-from core.runtime.resolver import resolveItem, extractItem
-from core.runtime.applyconditions import applyConditions
-from core.runtime.responsepolicy import responsePolicy
 from core.runtime.stopsignals import addStopSignal
 
 def buildPreRuntimeAttr(action, outputs, process):
+    from core.runtime import loadSave, readRow
+    from core.runtime.applyconditions import applyConditions
+    from core.runtime.resolver import resolveItem
+
     outputs = loadSave(action, outputs)
 
     resolved = resolveItem(action, outputs)
@@ -17,28 +16,31 @@ def buildPreRuntimeAttr(action, outputs, process):
     match, filter = applyConditions(action, resolved, outputs)
 
     if match:
-        outputs = addStopSignal(action, outputs, process)
+        logging.info(filter.get("print"))
+        addStopSignal(action, outputs, process)
     
     return outputs, resolved, match, filter
     
 
 def buildRuntimeAttr(action, outputs, resolved, response, process):
-    outputs = responsePolicy(response, action, outputs, resolved)
+    from core.runtime.resolver import extractItem
+    from core.runtime.responsepolicy import responsePolicy
+    from core.runtime.image import buildImage
+    from core.runtime import buildSelect, buildOutput, buildSave, saveRow, buildLog
 
-    #print(f"Response status code: {response.status_code} during {action.name}")
+    response = responsePolicy(response, action, outputs, resolved)
-    #print(f"Response content: {response.content} during {action.name}")
 
-    outputs = buildOutput(response, action, outputs)
+    buildOutput(response, action, outputs)
 
     if not response.ok:
-        outputs = addStopSignal(action, outputs, process)
+        addStopSignal(action, outputs, process)
         return outputs, response
 
-    resolved = extractItem(action, outputs, resolved)
+    extractItem(action, outputs, resolved)
 
-    outputs = buildImage(response, action, outputs)
+    buildImage(response, action, outputs)
     
-    action, outputs = buildSelect(action, outputs)
+    buildSelect(action, outputs)
 
     buildSave(action, outputs)
 

core/runtime/savedata.py

@@ -1,5 +1,6 @@
 import json
 import os
+import logging
 
 def buildSave(action, outputs):
     savedata_log = os.environ.get("LOGS_SAVE_DATA_ACCESS", "false").lower() == "true"
@@ -39,7 +40,7 @@ def buildSave(action, outputs):
                     pass
 
             if savedata_log:
-                print(f"Data saved by {action.name} to {action.savepath}")
+                logging.info(f"Data saved by {action.name} to {action.savepath}")
         return outputs
 
 def loadSave(action, outputs):
@@ -53,7 +54,7 @@ def loadSave(action, outputs):
             outputs[load_name] = load_data[load_name]
 
         if savedata_log:
-            print(f"Data loaded by {action.name} from {action.savepath}")
+            logging.info(f"Data loaded by {action.name} from {action.savepath}")
     return outputs
 
 def saveRow(action, outputs, resolved):
@@ -87,8 +88,8 @@ def saveRow(action, outputs, resolved):
                 elif "resolved" in item:
                     value = resolved[item["resolved"]]
             except KeyError as error:
-                print(f"The save data for {error} was not located in its respective dictionary during {action.name}")
+                logging.error(f"The save data for {error} was not located in its respective dictionary during {action.name}")
-                print("Bypassing database save")
+                logging.error("Bypassing database save")
                 return False
             
             values.append(value)
@@ -137,7 +138,7 @@ def saveRow(action, outputs, resolved):
                     database.execute(query, tuple(values))
 
         except sqlite3.IntegrityError as error:
-            print(f"Unable to complete database write during {action.name}: {error}")
+            logging.error(f"Unable to complete database write during {action.name}: {error}")
             return False
 
         #database.execute(
@@ -148,7 +149,7 @@ def saveRow(action, outputs, resolved):
         #database.commit()
         #database.close()
 
-        print(f"Successfully saved data to {table} during {action.name}")
+        logging.info(f"Successfully saved data to {table} during {action.name}")
 
     return True
 
@@ -206,8 +207,8 @@ def readRow(action, resolved, outputs):
             """,
             (value,)
         )
-        #print("DEBUG READROW TARGET ITEM")
+        logging.debug("DEBUG READROW TARGET ITEM")
-        #print(value)
+        logging.debug(value)
 
         result = cursor.fetchone()
 
@@ -216,15 +217,13 @@ def readRow(action, resolved, outputs):
             for column_index, output_name in exports.items():
                 dbtarget = result[column_index]
                 outputs[output_name] = dbtarget# result[column_index]
-                #print("DEBUGGING DATABASE LINE EXTRACTION IN READROW")
+                logging.debug("DEBUGGING DATABASE LINE EXTRACTION IN READROW")
-                #print(outputs[output_name])
+                logging.debug(outputs[output_name])
-                #print(dbtarget)
+                logging.debug(dbtarget)
 
             return resolved, outputs
         else:
-            print(f"Search target {value} not found in database table {table}")
+            logging.info(f"Search target {value} not found in database table {table}")
             return resolved, outputs
 
-        #print("READROW SEARCH RESULT")
-        #print(target)
     return resolved, outputs

core/runtime/select.py

@@ -39,12 +39,12 @@ def buildSelect(action, outputs):
                 if select_data is None:
                     print(f"No data selected during {action.name}")           
 
-            return action, outputs
+            return
         except KeyError as error:
             print(f"Error in selection of {error} during {action.name}")
 
         except TypeError as error:
             print(f"TypeError during select: {error} during {action.name}")
 
-    return action, outputs
+    return
     

core/runtime/stopsignals.py

@@ -17,4 +17,4 @@ def addStopSignal(action, outputs, operation):
     if hasattr(action, "depends"):
         outputs["processes"][operation.name]["stopsignals"].append(action.alias)
         #outputs["stopsignals"].append(action.alias)
-    return outputs
+    return

core/server/createlistener.py

@@ -1,10 +1,10 @@
 import json
+import logging
+
 from core.runtime.outputs import buildOutput
-from core.runtime.applyconditions import applyConditions
-from core.execution.operation import buildOperation
 from core.execution.executioncontract import executeContract
 from core.runtime.firewall import buildFirewall
-from core.execution.createthread import buildThread
+
 
 def buildListener(listener, self):
 
@@ -18,7 +18,7 @@ def buildListener(listener, self):
     if self.path.startswith(listener.acceptpath):
     
         #Prints can used buildlog for listener runtime
-        print("Matrix transaction received")
+        logging.info("Matrix transaction received")
 
         content_length = int(self.headers["Content-Length"])
 
@@ -27,27 +27,15 @@ def buildListener(listener, self):
 
         outputs = {}
 
-        outputs = buildOutput(data, listener, outputs)
+        buildOutput(data, listener, outputs)
-
-        #for item in outputs[listener.recursive["source"]]:
-
-            #match, filter = applyConditions(listener, item, outputs)
-
-            #if not match:
-        #        if "com.loxthelion.rmass_message_id" in item.get("content", {}):
-        #            print("Excluding previously imported items")
-        #            #print(item)
-        #            continue
-
-        allow, item, status_code = buildFirewall(listener, outputs)            
 
+        allow, status_code = buildFirewall(listener, outputs)
 
         if allow:
             #Print allowed items for logging purposes
-            print(outputs["firewall_allowed"])
+            logging.info(outputs["firewall_allowed"])
 
             #Process and forward to RMASS
-            #thread, lifecycles = buildThread(listener.execute, lifecycles)
             outputs, responses = executeContract(listener.execute, outputs, None)
 
 

core/server/server.py

@@ -1,16 +1,14 @@
 import os
 import json
+import logging
+
 from http.server import HTTPServer, BaseHTTPRequestHandler
 
 from .dispatcher import dispatchRequest
 from .createresponse import buildResponse
-from core.runtime.outputs import buildOutput
-from core.runtime.resolver import resolveItem
 from plugins.rmass.listeners import synapselistener
-from plugins.rmass.operations import rmassmessagepush
-from core.execution.action import buildAction
 from core.server.createlistener import buildListener
-from plugins.rmass.actions import pingsynapse
+#from plugins.rmass.actions import pingsynapse
 
 port = int(os.environ.get("SERVER_PORT", 19001))
 
@@ -104,7 +102,7 @@ class RequestHandler(BaseHTTPRequestHandler):
 def startServer():
     server = HTTPServer(("0.0.0.0", port), RequestHandler)
 
-    print(f"Server listening on port {port}")
+    logging.info(f"Server listening on port {port}")
     #outputs = {}
     #outputs, response = buildAction(pingsynapse, outputs)    
 

initialize.py

@@ -7,10 +7,21 @@ from core.execution.createthread import buildThread
 from plugins.rmass.monitors import rmassmonitor
 from plugins.rmass.actions import refresh_tokens
 
+from core.console import console
+from rich.logging import RichHandler
 
 logging.basicConfig(
     level=logging.WARNING,
-    format="%(levelname)s: %(message)s"
+    format="%(message)s",
+    handlers=[
+        RichHandler(
+            console=console,
+            show_time=False,
+            show_path=False
+        )
+        
+    ],
+    force=True
 )
 
 import login

plugins/rmass/monitors/rmassmonitor.py

@@ -59,7 +59,7 @@ filters = [
     },
 ]
 
-sleeptime = 30
+sleeptime = 15
 
 #prints = {
 #    "Status": "message",

plugins/rmass/operations/rmassmessagepush.py

@@ -1,7 +1,7 @@
 from plugins.rmass.actions import rmassmessagepush, savermassmessageid, synapsegetroomalias, synapsegetlastmessage
 from plugins.rmass.operations import rmassattachmentpushflow
 
-name = "RMASS Message Push"
+name = "Message Forwarding to RMASS"
 
 kind = "recursiveset"