Working resolver, working recursive logic with dynamic library creation

core/apirequest/path.py

@@ -6,9 +6,9 @@ def buildPath(path, outputs, resolved):
 
     for part in path:
         if isinstance(part, dict):
-            for key, val in part.items():      
+            #for key, val in part.items():      
                 #value = outputs[part["output"]][part["keys"]]
-                value = outputs[val]
+            value = resolved["path"]
                 #print("DEBUG path", value)
 
             #for key in part.get("keys", []):

core/apirequest/request.py

@@ -28,6 +28,4 @@ def buildRequest(definition, outputs, resolved):
 
         request["headers"] = headers
 
-    print(path)
-
     return request, outputs

core/execution/action/action.py

@@ -1,25 +1,12 @@
 from core.apirequest import executeRequest
-from core.runtime.savedata import loadSave, buildSave
+from core.runtime.runtimecontract import buildRuntimeAttr, buildPreRuntimeAttr
-from core.runtime.outputs import buildOutput
-from core.runtime.logs import buildLog
-from core.runtime.runtimecontract import buildRuntimeAttr
-from core.runtime.resolver import resolveItem
 
-#resolved ={}
-def buildAction (action, outputs):#, resolved):
 
-    if hasattr(action, "loaddata"):
+def buildAction (action, outputs):
-        outputs = loadSave(outputs, action)
+    outputs, resolved = buildPreRuntimeAttr(action, outputs)
-
-    resolved = {}
-    if hasattr(action, "resolve"):
-        resolved = resolveItem(action, outputs)
 
     response, outputs = executeRequest(action, outputs, resolved)
     
-    #print("DEBUG:", outputs)
-    #print(resolved)
-    
     outputs, response = buildRuntimeAttr(action, outputs, response)
 
     return outputs, response

core/execution/operation/operation.py

@@ -1,50 +1,27 @@
 import logging
-#from core import buildAction
-from core.runtime.savedata import buildSave, loadSave
-from core.runtime.logs import buildLog
-from core.runtime.outputs import buildOutput
 from core.runtime.managefailure import manageFailure
 from core.execution.action.action import buildAction
-from .skipstep import skipStep
+from core.runtime.skipstep import skipStep
-from core.runtime.resolver import resolveItem2
+from core.runtime.recursiveactions import recursiveAction
-from core.apirequest import executeRequest
 
 def buildOperation(definition):
     actions = definition.actions
-    #print("DEBUG", actions)
     responses = []
     outputs = {}
-    #resolved = {}
 
     for action in actions:
-        #try:
+
-        if hasattr(definition, "skipstep"):
         if skipStep(definition, action, outputs):
             continue
-        #if hasattr(definition, "resolve"):
+
-        #resolved = {}
+        if action == "recursiveset":
-        #if hasattr(action, "resolve"):
+            outputs = recursiveAction(definition, outputs)
-            #print(outputs)
+
-            #resolved = resolveItem(action, outputs)
-            #print(resolved)
-        if hasattr(action, "recursive"):
-            #outputs[action.recursive["target"]] = {}
-            for item in outputs[action.recursive["source"]]:
-                outputs = resolveItem2(action, item, outputs)#, resolved)
-                #print("Resolved value:", outputs)
-                #outputs[action.recursive["target"]] = outputs[action.recursive["source"]][][action.recursive["target"]]
-                #print(outputs[action.recursive["target"]])
-                outputs[action.recursive["target"]] = item
-                #print(outputs[action.recursive["target"]])
-                #target = outputs[action.recursive["source"]["target"]
-                outputs, response = buildAction(action, outputs)#, resolved)
-                #print(outputs)
-                #return outputs
         else:
             outputs, response = buildAction(action, outputs)
             responses.append(response)
   
         #except Exception as error:
             #outputs = manageFailure(error, action, outputs)
-    #print(outputs["message"])
+
     return outputs

core/runtime/__init__.py

@@ -2,7 +2,7 @@
 #from . import outputs
 #from . import managefailure
 #from . import savedata
-from .select import buildSelect, buildSelect2
+from .select import buildSelect
 from .outputs import buildOutput
 from .savedata import buildSave, loadSave
 from .logs import buildLog

core/runtime/logs.py

@@ -1,8 +1,7 @@
 import logging
 
 def buildLog (action, outputs, response):
-    #if response.status_code == 200:
+    if hasattr(action, "prints"):
-        #print(f"{action.name} successful")
         try:
             for print_name, print_key in action.prints.items():
                 print(
@@ -10,8 +9,12 @@ def buildLog (action, outputs, response):
                     f"{outputs[print_key]}",
                 )
 
+        except: (KeyError, TypeError)
+
+    if action.showstatus is True and response.status_code == 200:
+        print(f"{action.name} successful")
+
+    elif action.showstatus is True and response.status_code != 200:
+        print(f"{action.name} failed")
 
-    #else:
-        #print(f"{action.name} failed")
-    except: KeyError, TypeError
     return

core/runtime/outputs.py

@@ -3,7 +3,7 @@ def buildOutput(response, action, outputs):#, select_data):
         #response_data = select_data
     #else:
         #response_data = response.json()
-    
+    if hasattr(action, "outputs"):
         response_data = response.json()
         response_objects = [response_data]
 

core/runtime/recursiveactions.py

@@ -0,0 +1,33 @@
+from core.execution.action.action import buildAction
+
+def recursiveAction(definition, outputs):
+    recursiveset = definition.recursiveset["actions"]
+    for item in outputs[definition.recursiveset["source"]]:
+        outputs[definition.recursiveset["outputlabel"]] = item
+        #outputs["recursive"] = item
+
+        for job in recursiveset:
+            outputs, response = buildAction(job, outputs)
+        #outputs[action.recursive["target"]] = item
+
+        print(outputs["profile"])
+        print(outputs["messages"])
+    return outputs
+
+def recursiveActionStale(action, outputs):
+    #if hasattr(action, "recursive"):
+    #outputs[action.recursive["target"]] = {}
+    for item in outputs[action.recursive["source"]]:
+        #resolved = resolveItem(action, item)#, outputs)#, resolved)
+        #print("DEBUG Resolved value:", resolved)
+        #outputs[action.recursive["target"]] = outputs[action.recursive["source"]][][action.recursive["target"]]
+        #print(outputs[action.recursive["target"]])
+
+        #Target may be phased out here. its not quite necessary after resolve
+        outputs[action.recursive["target"]] = item
+
+        #print(outputs[action.recursive["target"]])
+        #target = outputs[action.recursive["source"]["target"]
+        outputs, response = buildAction(action, outputs)#, resolved)
+        #print(outputs)
+    return outputs

core/runtime/resolver.py

@@ -1,23 +1,24 @@
 def resolveItem(action, item):
     resolved = {}
-    label = action.resolve["label"]
+    if hasattr(action, "resolve"):
-    targetkeys = action.resolve["targetkeys"]
+    #label = action.resolve["label"]
+    #targetkeys = action.resolve["targetkeys"]
     #print(item)
-    target = item
-    
-    for key in targetkeys:
-        target = target[key]
-
-    resolved[label] = target
-
-    #for label, keys in action.resolve.items():
     #target = item
     
-        #print(target)
+    #for key in targetkeys:
-        #for key in keys:
     #    target = target[key]
 
     #resolved[label] = target
+
+        for label, keys in action.resolve.items():
+            target = item
+            #print(item)
+        #print(target)
+            for key in keys:
+                target = target[key]
+
+            resolved[label] = target
             #print(resolved[label])
 
     return resolved
@@ -47,3 +48,28 @@ def resolveItem2(action, item, outputs):
     outputs[label] = target
     #print("Resolver DEBUG", target, outputs[label])
     return outputs
+
+def resolveItemBackup(action, item):
+    resolved = {}
+    if hasattr(action, "resolve"):
+    #label = action.resolve["label"]
+    #targetkeys = action.resolve["targetkeys"]
+    #print(item)
+    #target = item
+    
+    #for key in targetkeys:
+    #    target = target[key]
+
+    #resolved[label] = target
+
+        for label, keys in action.resolve.items():
+            target = item
+            #print(item)
+        #print(target)
+            for key in keys:
+                target = target[key]
+
+            resolved[label] = target
+            print(resolved[label])
+
+    return resolved

core/runtime/runtimecontract.py

@@ -1,41 +1,21 @@
-from core.runtime import buildSelect, buildSelect2, buildOutput, buildSave, buildLog
+from core.runtime import buildSelect, buildOutput, buildSave, loadSave, buildLog
-from core.runtime.select import buildSelect3
+#from core.runtime.select import buildSelect3
-#select_data = {}
+from core.runtime.resolver import resolveItem
+
+def buildPreRuntimeAttr(action, outputs):
+    outputs = loadSave(action, outputs)
+
+    resolved = resolveItem(action, outputs)
+
+    return outputs, resolved
 
 def buildRuntimeAttr(action, outputs, response):
-    #select_data = {}
+    outputs = buildOutput(response, action, outputs)
-    #response_data = response.json()
     
-    #if hasattr(action, "select"):
+    select_data, action, outputs = buildSelect(action, outputs)
-        #select_data, action, outputs = buildSelect(response, action, outputs)
-    #else:
-        #select_data = "not applied"
-    #print("BEFORE buildOutput:", type(outputs))
-    if hasattr(action, "outputs"):
-        outputs = buildOutput(response, action, outputs)#, select_data)
  
-    if hasattr(action, "select"):
+    buildSave(action, outputs)
-        select_data, action, outputs = buildSelect3(action, outputs)
-    #print("AFTER buildOutput:", type(outputs))
-    #print("DEBUG buildSave")
-    #print("outputs type:", type(outputs))
-    #print("action type:", type(action))
-    #print("outputs keys:", list(outputs) if isinstance(outputs, dict) else "NOT A DICT")
 
-    if hasattr(action, "savedata"):
-        buildSave(outputs, action)
-
-    if hasattr(action, "prints"):
     buildLog(action, outputs, response)
 
-    #if hasattr(action, "conversion"):
-        #conversions = action.conversion.items()
-        #for key in action.conversion.items():
-           #print(f"{action.conversion.items[key]}")
-           #convertedoutput = conversions[key]
-           #outputs[conversions[key]] = convertedoutput
-
-    if action.showstatus is True and response.status_code == 200:
-        print(f"{action.name} successful")
-
     return outputs, response

core/runtime/savedata.py

@@ -1,6 +1,7 @@
 import json
 
-def buildSave(outputs, action):
+def buildSave(action, outputs):
+    if hasattr(action, "savedata"):
         save_data = {}
 
         for save_name in action.savedata:
@@ -9,17 +10,16 @@ def buildSave(outputs, action):
             with open(f"{action.savepath}", "w") as save_file:
                 json.dump(save_data, save_file)
 
-        #print(f"{save_name} saved by {action.name}")
         print(f"Data saved by {action.name} to {action.savepath}")
     return outputs
 
-def loadSave(outputs, action):
+def loadSave(action, outputs):
+    if hasattr(action, "loaddata"):
         with open(f"{action.savepath}", "r") as save_file:
             load_data = json.load(save_file)
 
         for load_name in action.loaddata:
             outputs[load_name] = load_data[load_name]
-        #print(f"{load_name} loaded by {action.name}")
 
         print(f"Data loaded by {action.name} from {action.savepath}")
     return outputs

core/runtime/select.py

@@ -1,75 +1,9 @@
-def buildSelect(response, action, outputs):
+def buildSelect(action, outputs):
-    #if hasattr(action, "select"):
+#***THIS IS THE CANONICAL BUILD***
-
-    response_data = response.json()
-    #selection = action.select
-    #print(response_data)
-
-    select_data ={}
-    for item, index in action.select.items():
-        select_data[item] = response_data[item][index]
-
-        print(select_data)
-
-    #if select_data == None:
-    #    selection = "not applied"
-    #else:
-        selection = select_data
-
-    return selection, action, outputs
-
-
-def buildSelect2(action, outputs):
-    #print(outputs)
     select_data = []
-    target = outputs[action.select["source"]]
-    selectmethod = action.select["selectmethod"]
-    targetkey = action.select["key"]
-    targetvalue = action.select["value"]
 
-    if isinstance(targetvalue, dict):
+    if hasattr(action, "select"):
-        for resolvekey, resolveval in targetvalue.items():
-            print(resolvekey)
-            print(resolveval)
-                    #outputtarget = outputs[resolvekey]
-                    #for thing in resolveval:
-            outputtarget = outputs[resolvekey][resolveval[0]][resolveval[1]]
-            print (outputtarget)
-            targetvalue = outputtarget
-    #for item in target:
-    for item in target:
-        #print (item)    
-        for key, value in item.items():
-            #if isinstance(targetvalue, dict):
-                #for resolvekey, resolveval in targetvalue.items():
-                    #print(resolvekey)
-                    #print(resolveval)
-                    #outputtarget = outputs[resolvekey]
-                    #for thing in resolveval:
-                    #outputtarget = outputs[resolvekey][resolveval[0]][resolveval[1]]
-                    #print (outputtarget)
-                    #outputtarget = targetvalue
-            if selectmethod == "exclude":    
-                if targetkey == key and item[key] != targetvalue:
-                    select_data.append(item)
-                    #print(item)
-            if selectmethod == "include":    
-                if targetkey == key and item[key] == targetvalue:
-                    select_data.append(item)
-                    #print(item)
-    outputs[action.select["source"]] = select_data
-    #print(outputs)
-    return select_data, action, outputs
-    #for item in outputs[output_name]:
-        #for key, value in action.select.include.items():
-            #if key in item and item[key] == value:
-                #select_data.append(item)
-
-def buildSelect3(action, outputs):
-    #print(outputs)
         try:
-        select_data = []
-
             target = outputs[action.select["source"]]
             includekey = action.select["include"]["key"]
             includevalue = action.select["include"]["value"]
@@ -90,76 +24,11 @@ def buildSelect3(action, outputs):
                 if include_match and exclude_match:
                     select_data.append(item)
 
-     
             outputs[action.select["source"]] = select_data
-        print(select_data)
+            #print(select_data)            
+
             return select_data, action, outputs
         except (KeyError, TypeError):
+            select_data = ["no selection"]
     return select_data, action, outputs
     
-def buildSelect4(action, outputs):
-    #print(outputs)
-    select_data = []
-
-    target = outputs[action.select["source"]]
-    #selectmethod = action.select["selectmethod"]
-    includekey = action.select["include"]["key"]
-    includevalue = action.select["include"]["value"]
-    excludekey = action.select["exclude"]["key"]
-    excludevalue = action.select["exclude"]["value"]
-    #if isinstance(targetvalue, dict):
-        #for resolvekey, resolveval in targetvalue.items():
-            #print(resolvekey)
-            #print(resolveval)
-                    #outputtarget = outputs[resolvekey]
-                    #for thing in resolveval:
-            #outputtarget = outputs[resolvekey][resolveval[0]][resolveval[1]]
-            #print (outputtarget)
-            #targetvalue = outputtarget
-    #for item in target:
-    #include_match = True
-    #exclude_match = True
-    
-    for item in target:
-        include_match = True
-        exclude_match = True
-
-        if includekey:
-            include_match = item[includekey] == includevalue
-
-        if excludekey:
-            exclude_match = item[excludekey] != excludevalue
-            
-        if include_match and exclude_match:
-            select_data.append(item)
-        #print (item)    
-        #for key, value in item.items():
-            #if isinstance(targetvalue, dict):
-                #for resolvekey, resolveval in targetvalue.items():
-                    #print(resolvekey)
-                    #print(resolveval)
-                    #outputtarget = outputs[resolvekey]
-                    #for thing in resolveval:
-                    #outputtarget = outputs[resolvekey][resolveval[0]][resolveval[1]]
-                    #print (outputtarget)
-                    #outputtarget = targetvalue
-            #if selectmethod == "exclude": 
-        #if item[excludekey] != excludevalue:# and item[includekey] == includevalue:
-            #if includekey == key and item[key] == includevalue:
-        #if item[excludekey] == excludevalue:
-            #continue
-            #select_data.append(item)
-        #if item[includekey] == includevalue:
-            #if includekey == key and item[key] == includevalue:
-                #continue
-            #select_data.append(item)
-            #if excludekey == key and item[key] != excludevalue:
-                #select_data.append(item)
-                #print(item)
-            #if selectmethod == "include":    
-            #if excludekey == key and item[key] == excludevalue:
-                    #select_data.append(item)
-                    #print(item)
-    outputs[action.select["source"]] = select_data
-    print(select_data)
-    return select_data, action, outputs

core/runtime/skipstep.py

@@ -0,0 +1,21 @@
+def skipStep(definition, action, outputs):
+    if hasattr(definition, "skipstep"):
+        try:
+            if action in definition.skipstep:
+                condition = definition.skipstep[action]
+        
+                #if outputs.get([condition["output"]]) is None:
+                    #return False
+                    #current_value = 1
+                #else:
+                current_value = outputs[condition["output"]]
+                #current_value = 1
+
+                if condition["operator"] == ">=":
+                    if current_value >= condition["value"]:
+                        print(f"Skipping {action.name}")
+                        return True
+
+        except KeyError:
+        #outputs = manageFailure(error)
+            return False

plugins/rmass/actions/getmessage.py

@@ -1,24 +1,22 @@
 import os
 
 
-name = "Get message"
+name = "Get messages"
 
 kind = "action"
 
-#id = "getmessage"
-
 showstatus = True
 
 method = "GET"
 
 resolve = {
-    "targetkeys": ["id"],
+    "path": ["messageheaders", "id"],
-    "label": "path",
+    #"label": "path",
 }
 
 path = [
     "https://rentmasseur.com/api/v1/mailbox/",
-    {"suffix": "path"}
+    {"resolved": "path"},
     #{
     #    "output": "id",
     #    #"keys": [0, "id"]
@@ -35,20 +33,13 @@ headers = {
     },
 }      
 
-recursive = {
-    "source": "emails",
-    #Transformation target. dynamic target to be resolved in each iteration
-    "target": "id"
-}
-
-
 outputs = {
-    "message": "messages",
+    "messages": "messages",
     "profile": "profileData",
 }
 
 select = {
-    "source": "message",
+    "source": "messages",
     "include": {
         "key": "seen",
         "value": False,
@@ -58,6 +49,4 @@ select = {
         "value": 238302,
     }
 }
-#prints = {
+
-#    "message": "messages"
-#}

plugins/rmass/actions/synapsepush.py

@@ -0,0 +1,52 @@
+import os
+
+name = "Synapse Message Push"
+
+kind = "action"
+
+showstatus = True
+
+astoken = os.environ["AS_TOKEN"]
+
+resolve = {
+    "sender": ["message", "senderId"],
+    #"messageid": ["message", "id"],
+    "username": ["profile", "username"],
+    "location": ["profile", "location"],
+}
+keys = [
+    {
+        "keytitle": "type",
+        "keyvalue": ,
+    },
+    {
+        "keytitle": "username",
+        "keyvalue": {outputs: senderId},
+    },
+    {
+        "keytitle": "csrf",
+        "keyvalue": {
+            "outputs": "csrf_token",
+        },
+    },
+    {
+        "keytitle": "remember",
+        "keyvalue": 1,
+    },
+]
+
+method = "POST"
+
+path = "https://rentmasseur.com/api/v1/login"
+
+outputs = {
+    "access_token": "accessToken",
+    "refresh_token": "refreshToken",
+}
+
+savedata = [
+    "access_token",
+    "refresh_token",
+]
+
+savepath = os.environ["TOKEN_STORE"]

plugins/rmass/operations/pullmessagelist.py

@@ -1,4 +1,4 @@
-from plugins.rmass.actions import pullmessagelist, refresh_tokens, getmessage
+from plugins.rmass.actions import pullmessagelist, refresh_tokens, getmessage, checkavailable
 
 name = "Get Message List"
 
@@ -7,20 +7,13 @@ kind = "operation"
 actions = [
     refresh_tokens,
     pullmessagelist,
-    getmessage,
+    "recursiveset",
     #setavailable,
 ]
 
-#sleeptime = 10
+recursiveset = {
+    "source": "emails",
+    "outputlabel": "messageheaders",
+    "actions": [getmessage],
+}        
 
-#skipstep = {
-#    setavailable: {
-#        "output": "validTo",
-#        "operator": ">=",
-#        "value": 3600,
-#    }
-#}
-
-#role = "activate"
-
-#sleeptime = 180

tokens.json

@@ -1 +1 @@
-{"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJleHAiOjE3OTE5MTgyMDEsIm5iZiI6MTc5MTkxODAyMSwic3ViIjoiMjM4MzAyIiwidXNlcm5hbWUiOiJYeHhvdGlxIiwidXNlclR5cGUiOjEsInN0YWdlTmFtZSI6Ilh4eG90aXEiLCJhcHByb3ZlZCI6MSwibG9jYXRpb25JZCI6NDcyLCJzZWFyY2hDaXR5IjoiYXRsYW50YSIsImNvdW50cnlTaG9ydCI6IlVTIiwic2Vzc2lvbkNvdW50cnlDb250aW5lbnQiOiJOQSIsImlzRnJvemVuIjowLCJsb2NhdGlvbkxhYmVsIjoiQXRsYW50YSwgR0EiLCJpc3MiOiJodHRwczovL3JlbnRtYXNzZXVyLmNvbSIsImlhdCI6MTc5MTkxODA4MX0.Y5Ej40c3Ji_-WLShpJkRSl3wF4XGxrLhE_rYMWxQmeDPHK03mv8bWX_7A7zmjjk3cYrwlBM5odn_eJWSaCI_Lg", "refresh_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJleHAiOjE3OTQ1MTAwODEsIm5iZiI6MTc5MTkxODAyMSwianRpIjoiQ0NmdW9xTFdsUmZpais1c1VxU2FudFRVb0xZVkd3ZG9NQkRYY1ZnV0pzdz0iLCJpc3MiOiJodHRwczovL3JlbnRtYXNzZXVyLmNvbSIsImlhdCI6MTc5MTkxODA4MSwic3ViIjoiMjM4MzAyIn0.oWa2PwSyh110MOJ8ypsRCtiXNQWosNv4T_UZ9m0shUjsFTX0Qtr-bSnc7mS8-5TSW-szEnLHT89-T8KzNZApYg"}
+{"access_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJleHAiOjE3OTE5NTY3NTUsIm5iZiI6MTc5MTk1NjU3NSwic3ViIjoiMjM4MzAyIiwidXNlcm5hbWUiOiJYeHhvdGlxIiwidXNlclR5cGUiOjEsInN0YWdlTmFtZSI6Ilh4eG90aXEiLCJhcHByb3ZlZCI6MSwibG9jYXRpb25JZCI6NDcyLCJzZWFyY2hDaXR5IjoiYXRsYW50YSIsImNvdW50cnlTaG9ydCI6IlVTIiwic2Vzc2lvbkNvdW50cnlDb250aW5lbnQiOiJOQSIsImlzRnJvemVuIjowLCJsb2NhdGlvbkxhYmVsIjoiQXRsYW50YSwgR0EiLCJpc3MiOiJodHRwczovL3JlbnRtYXNzZXVyLmNvbSIsImlhdCI6MTc5MTk1NjYzNX0.G1vCD8bSnaobc71onjgABLIlyoG1LOez4PRbdVSzcSe4aP-HefaU97gky9df18j-msaruFdFpuqWHozsZezspw", "refresh_token": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzUxMiJ9.eyJleHAiOjE3OTQ1NDg2MzUsIm5iZiI6MTc5MTk1NjU3NSwianRpIjoiVGFUalhWcm5oUDY5bEdIYzR4OGdKemU1TFpBR2FwMUU3Uk5UMitGY2Rzbz0iLCJpc3MiOiJodHRwczovL3JlbnRtYXNzZXVyLmNvbSIsImlhdCI6MTc5MTk1NjYzNSwic3ViIjoiMjM4MzAyIn0.vZUZ0XqogvKCm8LPnoaPC-zNBwdafJCiSyPM6mv0L2FsM6NhOv4a2mVpHHifSa2ZIafse1MMlwSpMzJl1q7a5A"}