added broserless /function support

scraping.go

@@ -244,6 +244,14 @@ func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, d
 				{
 					getFilterResultBrowserlessURLs(filter, web.urlCache, debug)
 				}
+			case "func":
+				{
+					getBrowserlessFunctionResult(filter)
+				}
+			case "funcs":
+				{
+					getBrowserlessFunctionResults(filter)
+				}
 			}
 		}
 	case filter.Type == "echo":
@@ -384,6 +392,7 @@ func getBrowserlessURLContent(filter *Filter, fetchURL string) (string, error) {
 		filter.log("Could not marshal url:", err)
 		return "", err
 	}
+	browserlessURL = browserlessURL + "/content"
 	resp, err := http.Post(browserlessURL, "application/json", bytes.NewBuffer(jsn))
 	if err != nil {
 		log.Println("Could not get browserless response content:", err)
@@ -399,6 +408,74 @@ func getBrowserlessURLContent(filter *Filter, fetchURL string) (string, error) {
 	return string(body), nil
 }
 
+func getBrowserlessFunctionResult(filter *Filter) {
+	result, err := getBrowserlessFunctionContent(filter, "")
+	if err != nil {
+		log.Println(err)
+		filter.log(err)
+		return
+	}
+	filter.Results = append(filter.Results, result)
+}
+
+func getBrowserlessFunctionResults(filter *Filter) {
+	for _, parent := range filter.Parents {
+		for _, result := range parent.Results {
+			fetchURL := result
+
+			str, err := getBrowserlessFunctionContent(filter, fetchURL)
+			if err != nil {
+				log.Println(err)
+				filter.log(err)
+				continue
+			}
+			filter.Results = append(filter.Results, str)
+		}
+	}
+}
+
+type BrowserlessContext struct {
+	Result string `json:"result"`
+}
+
+func getBrowserlessFunctionContent(filter *Filter, result string) (string, error) {
+	if !viper.IsSet("browserless.url") {
+
+		return "", errors.New("browserless.url not set")
+	}
+	browserlessURL := viper.GetString("browserless.url")
+	if filter.Var2 == nil {
+		return "", errors.New("filter.Var2 == nil")
+	}
+	code := *filter.Var2
+	data := struct {
+		Code     string             `json:"code"`
+		Context  BrowserlessContext `json:"context"`
+		Detached bool               `json:"detached"`
+	}{
+		Code:     code,
+		Context:  BrowserlessContext{Result: result},
+		Detached: false,
+	}
+	jsn, err := json.Marshal(data)
+	if err != nil {
+		return "", err
+	}
+
+	browserlessURL = browserlessURL + "/function"
+	resp, err := http.Post(browserlessURL, "application/json", bytes.NewBuffer(jsn))
+	if err != nil {
+		return "", err
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", err
+	}
+
+	return string(body), nil
+}
+
 func getFilterResultXPath(filter *Filter) {
 	selectType := "node"
 	if filter.Var2 != nil {

static/edit.js

@@ -626,54 +626,14 @@ function onTypeChange(node) {
             gurlsOption.value = "gurls";
             gurlsOption.innerHTML = "Get URLs";
             browserlessSelect.appendChild(gurlsOption);
-            var startOption = document.createElement("option");
+            var funcOption = document.createElement("option");
-            startOption.value = "start";
+            funcOption.value = "func";
-            startOption.innerHTML = "Start Session";
+            funcOption.innerHTML = "Function";
-            browserlessSelect.appendChild(startOption);
+            browserlessSelect.appendChild(funcOption);
-            var endOption = document.createElement("option");
+            var funcsOption = document.createElement("option");
-            endOption.value = "end";
+            funcsOption.value = "funcs";
-            endOption.innerHTML = "End Session";
+            funcsOption.innerHTML = "Function on results";
-            browserlessSelect.appendChild(endOption);
+            browserlessSelect.appendChild(funcsOption);
-            var gotoOption = document.createElement("option");
-            gotoOption.value = "go";
-            gotoOption.innerHTML = "Goto URL";
-            browserlessSelect.appendChild(gotoOption);
-            var cssOption = document.createElement("option");
-            cssOption.value = "css";
-            cssOption.innerHTML = "Click CSS Element";
-            browserlessSelect.appendChild(cssOption);
-            var xpathOption = document.createElement("option");
-            xpathOption.value = "xpath";
-            xpathOption.innerHTML = "Click XPath Element";
-            browserlessSelect.appendChild(xpathOption);
-            var authOption = document.createElement("option");
-            authOption.value = "auth";
-            authOption.innerHTML = "Authenticate";
-            browserlessSelect.appendChild(authOption);
-            var selectOption = document.createElement("option");
-            selectOption.value = "select";
-            selectOption.innerHTML = "Set Dropdown";
-            browserlessSelect.appendChild(selectOption);
-            var inputOption = document.createElement("option");
-            inputOption.value = "input";
-            inputOption.innerHTML = "Set Input";
-            browserlessSelect.appendChild(inputOption);
-            var keyOption = document.createElement("option");
-            keyOption.value = "key";
-            keyOption.innerHTML = "Press Key";
-            browserlessSelect.appendChild(keyOption);
-            var waitOption = document.createElement("option");
-            waitOption.value = "wait";
-            waitOption.innerHTML = "Wait";
-            browserlessSelect.appendChild(waitOption);
-            var lineOption = document.createElement("option");
-            lineOption.value = "line";
-            lineOption.innerHTML = "Raw Line";
-            browserlessSelect.appendChild(lineOption);
-            var rawOption = document.createElement("option");
-            rawOption.value = "raw";
-            rawOption.innerHTML = "Raw Function";
-            browserlessSelect.appendChild(rawOption);
             var1Div.appendChild(browserlessSelect);
             if (var1Value == "") {
                 browserlessSelect.value = "gurl";
@@ -754,7 +714,8 @@ function onTypeChange(node) {
                 }
                 finally { if (e_2) throw e_2.error; }
             }
-            var1Div.appendChild(snippetDiv);
+            var2Label.innerHTML = "Snippets";
+            var2Div.appendChild(snippetDiv);
             var var2Input = document.createElement("input");
             var2Input.name = "var2";
             var2Input.id = "var2Input";
@@ -820,7 +781,6 @@ function onConditionChange(node) {
     var var1Value = "";
     var var2Value = "";
     var var3Value = "";
-    console.log(node, var1Input.value);
     if (node != null) {
         // @ts-ignore
         var1Value = node.meta.var1;
@@ -903,10 +863,10 @@ function onBrowserlessChange(node) {
     var var3Input = document.getElementById("var3Input");
     var var3Label = document.getElementById("var3Label");
     var var3Div = document.getElementById("var3Div");
+    var3Div.innerHTML = "";
     var var1Value = "";
     var var2Value = "";
     var var3Value = "";
-    console.log(node, var1Input.value);
     if (node != null) {
         // @ts-ignore
         var1Value = node.meta.var1;
@@ -932,6 +892,78 @@ function onBrowserlessChange(node) {
             break;
         }
         case "gurls": {
+            var var2Input_4 = document.createElement("input");
+            var2Input_4.name = "var2";
+            var2Input_4.id = "var2Input";
+            var2Input_4.value = "";
+            var2Input_4.disabled = true;
+            var2Input_4.classList.add("form-control");
+            var2Label.innerHTML = "-";
+            var2Div.appendChild(var2Input_4);
+            break;
+        }
+        case "func": {
+            var var2Input_5 = document.createElement("textarea");
+            var2Input_5.name = "var2Input";
+            var2Input_5.id = "var2Input";
+            var2Input_5.value = "module.exports = async ({ page, context }) => {\n  await page.goto(\"https://192.168.178.254:8000\");\n\n  const data = await page.content();\n\n  return {\n    data,\n    type: 'text/plain', // 'application/html' 'application/json'\n  };\n};";
+            var2Input_5.classList.add("form-control");
+            var2Input_5.rows = 15;
+            var2Label.innerHTML = "Code";
+            var2Div.appendChild(var2Input_5);
+            if (var2Value != "") {
+                var2Input_5.value = var2Value;
+            }
+            var var3Input_1 = document.createElement("input");
+            var3Input_1.type = "hidden";
+            var3Input_1.id = "var3Input";
+            var3Input_1.name = "var3Input";
+            var3Div.appendChild(var3Input_1);
+            var3Label.innerHTML = "Help";
+            var helpLink1 = document.createElement("a");
+            helpLink1.href = "https://www.browserless.io/docs/function";
+            helpLink1.innerHTML = "Browserless /Funcion";
+            helpLink1.target = "_blank";
+            helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink1);
+            var helpLink2 = document.createElement("a");
+            helpLink2.href = "https://pptr.dev/api/puppeteer.page";
+            helpLink2.innerHTML = "Puppeteer Page";
+            helpLink2.target = "_blank";
+            helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink2);
+            break;
+        }
+        case "funcs": {
+            var var2Input_6 = document.createElement("textarea");
+            var2Input_6.name = "var2Input";
+            var2Input_6.id = "var2Input";
+            var2Input_6.value = "module.exports = async ({ page, context }) => {\n  const { result } = context;\n  await page.goto(result);\n\n  const data = await page.content();\n\n  return {\n    data,\n    type: 'text/plain', // 'application/html' 'application/json'\n  };\n};";
+            var2Input_6.classList.add("form-control");
+            var2Input_6.rows = 15;
+            var2Label.innerHTML = "Code";
+            var2Div.appendChild(var2Input_6);
+            if (var2Value != "") {
+                var2Input_6.value = var2Value;
+            }
+            var var3Input_2 = document.createElement("input");
+            var3Input_2.type = "hidden";
+            var3Input_2.id = "var3Input";
+            var3Input_2.name = "var3Input";
+            var3Div.appendChild(var3Input_2);
+            var3Label.innerHTML = "Help";
+            var helpLink1 = document.createElement("a");
+            helpLink1.href = "https://www.browserless.io/docs/function";
+            helpLink1.innerHTML = "Browserless /Funcion";
+            helpLink1.target = "_blank";
+            helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink1);
+            var helpLink2 = document.createElement("a");
+            helpLink2.href = "https://pptr.dev/api/puppeteer.page";
+            helpLink2.innerHTML = "Puppeteer Page";
+            helpLink2.target = "_blank";
+            helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink2);
             break;
         }
     }

static/edit.ts

@@ -614,65 +614,15 @@ function onTypeChange(node: DiagramNode | null = null){
             gurlsOption.innerHTML = "Get URLs";
             browserlessSelect.appendChild(gurlsOption);
             
-            let startOption = document.createElement("option");
+            let funcOption = document.createElement("option");
-            startOption.value = "start"
+            funcOption.value = "func"
-            startOption.innerHTML = "Start Session";
+            funcOption.innerHTML = "Function";
-            browserlessSelect.appendChild(startOption);
+            browserlessSelect.appendChild(funcOption);
             
-            let endOption = document.createElement("option");
+            let funcsOption = document.createElement("option");
-            endOption.value = "end"
+            funcsOption.value = "funcs"
-            endOption.innerHTML = "End Session";
+            funcsOption.innerHTML = "Function on results";
-            browserlessSelect.appendChild(endOption);
+            browserlessSelect.appendChild(funcsOption);
-            
-            let gotoOption = document.createElement("option");
-            gotoOption.value = "go"
-            gotoOption.innerHTML = "Goto URL";
-            browserlessSelect.appendChild(gotoOption);
-            
-            let cssOption = document.createElement("option");
-            cssOption.value = "css"
-            cssOption.innerHTML = "Click CSS Element";
-            browserlessSelect.appendChild(cssOption);
-
-            let xpathOption = document.createElement("option");
-            xpathOption.value = "xpath"
-            xpathOption.innerHTML = "Click XPath Element";
-            browserlessSelect.appendChild(xpathOption);
-
-            let authOption = document.createElement("option");
-            authOption.value = "auth"
-            authOption.innerHTML = "Authenticate";
-            browserlessSelect.appendChild(authOption);
-
-            let selectOption = document.createElement("option");
-            selectOption.value = "select"
-            selectOption.innerHTML = "Set Dropdown";
-            browserlessSelect.appendChild(selectOption);
-
-            let inputOption = document.createElement("option");
-            inputOption.value = "input"
-            inputOption.innerHTML = "Set Input";
-            browserlessSelect.appendChild(inputOption);
-
-            let keyOption = document.createElement("option");
-            keyOption.value = "key"
-            keyOption.innerHTML = "Press Key";
-            browserlessSelect.appendChild(keyOption);
-
-            let waitOption = document.createElement("option");
-            waitOption.value = "wait"
-            waitOption.innerHTML = "Wait";
-            browserlessSelect.appendChild(waitOption);
-
-            let lineOption = document.createElement("option");
-            lineOption.value = "line"
-            lineOption.innerHTML = "Raw Line";
-            browserlessSelect.appendChild(lineOption);
-
-            let rawOption = document.createElement("option");
-            rawOption.value = "raw"
-            rawOption.innerHTML = "Raw Function";
-            browserlessSelect.appendChild(rawOption);
             
             var1Div.appendChild(browserlessSelect);
 
@@ -885,7 +835,8 @@ end
                 gap.classList.add("m-1", "xs");
                 snippetDiv.appendChild(gap);
             }
-            var1Div.appendChild(snippetDiv);
+            var2Label.innerHTML = "Snippets";
+            var2Div.appendChild(snippetDiv);
 
             let var2Input = document.createElement("input");
             var2Input.name = "var2";
@@ -954,7 +905,6 @@ function onConditionChange(node: DiagramNode | null = null){
     let var1Value = "";
     let var2Value = "";
     let var3Value = "";
-    console.log(node, var1Input.value)
     if (node != null){
         // @ts-ignore
         var1Value = node.meta.var1;
@@ -1029,11 +979,11 @@ function onBrowserlessChange(node: DiagramNode | null = null){
     let var3Input = document.getElementById("var3Input") as HTMLInputElement;
     let var3Label = document.getElementById("var3Label") as HTMLLabelElement;
     let var3Div = document.getElementById("var3Div") as HTMLDivElement;
+    var3Div.innerHTML = "";
 
     let var1Value = "";
     let var2Value = "";
     let var3Value = "";
-    console.log(node, var1Input.value)
     if (node != null){
         // @ts-ignore
         var1Value = node.meta.var1;
@@ -1056,10 +1006,109 @@ function onBrowserlessChange(node: DiagramNode | null = null){
             var2Input.value = var2Value;
             var2Input.classList.add("form-control")
             var2Label.innerHTML = "URL";
-            var2Div.appendChild(var2Input)
+            var2Div.appendChild(var2Input);
             break;
         }
         case "gurls": {
+            let var2Input = document.createElement("input");
+            var2Input.name = "var2";
+            var2Input.id = "var2Input";
+            var2Input.value = "";
+            var2Input.disabled = true;
+            var2Input.classList.add("form-control")
+            var2Label.innerHTML = "-";
+            var2Div.appendChild(var2Input);
+            break;
+        }
+        case "func": {
+            let var2Input = document.createElement("textarea");
+            var2Input.name = "var2Input";
+            var2Input.id = "var2Input";
+            var2Input.value = `module.exports = async ({ page, context }) => {
+  await page.goto("https://192.168.178.254:8000");
+
+  const data = await page.content();
+
+  return {
+    data,
+    type: 'text/plain', // 'application/html' 'application/json'
+  };
+};`;
+            var2Input.classList.add("form-control");
+            var2Input.rows = 15;
+            var2Label.innerHTML = "Code";
+            var2Div.appendChild(var2Input);
+
+            if (var2Value != ""){
+                var2Input.value = var2Value;
+            }
+
+            let var3Input = document.createElement("input");
+            var3Input.type = "hidden";
+            var3Input.id = "var3Input";
+            var3Input.name = "var3Input";
+            var3Div.appendChild(var3Input);
+
+            var3Label.innerHTML = "Help";
+            let helpLink1 = document.createElement("a");
+            helpLink1.href = "https://www.browserless.io/docs/function";
+            helpLink1.innerHTML = "Browserless /Funcion";
+            helpLink1.target = "_blank";
+            helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink1);
+
+            let helpLink2 = document.createElement("a");
+            helpLink2.href = "https://pptr.dev/api/puppeteer.page";
+            helpLink2.innerHTML = "Puppeteer Page";
+            helpLink2.target = "_blank";
+            helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink2);
+            break;
+        }
+        case "funcs":{
+            let var2Input = document.createElement("textarea");
+            var2Input.name = "var2Input";
+            var2Input.id = "var2Input";
+            var2Input.value = `module.exports = async ({ page, context }) => {
+  const { result } = context;
+  await page.goto(result);
+
+  const data = await page.content();
+
+  return {
+    data,
+    type: 'text/plain', // 'application/html' 'application/json'
+  };
+};`;
+            var2Input.classList.add("form-control");
+            var2Input.rows = 15;
+            var2Label.innerHTML = "Code";
+            var2Div.appendChild(var2Input);
+
+            if (var2Value != ""){
+                var2Input.value = var2Value;
+            }
+
+            let var3Input = document.createElement("input");
+            var3Input.type = "hidden";
+            var3Input.id = "var3Input";
+            var3Input.name = "var3Input";
+            var3Div.appendChild(var3Input);
+
+            var3Label.innerHTML = "Help";
+            let helpLink1 = document.createElement("a");
+            helpLink1.href = "https://www.browserless.io/docs/function";
+            helpLink1.innerHTML = "Browserless /Funcion";
+            helpLink1.target = "_blank";
+            helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink1);
+
+            let helpLink2 = document.createElement("a");
+            helpLink2.href = "https://pptr.dev/api/puppeteer.page";
+            helpLink2.innerHTML = "Puppeteer Page";
+            helpLink2.target = "_blank";
+            helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs");
+            var3Div.appendChild(helpLink2);
             break;
         }
     }

todo.md

@@ -5,22 +5,5 @@
     - util.go
     - edit.ts
     - diagram.ts
-- browserless function filters
-    - ~~get url~~
-    - ~~get urls~~
-    - session
-        - start
-        - end
-    - goto url
-    - click elem
-        - xpath
-        - css
-    - basic auth
-    - set select
-    - set text input
-    - press key
-    - wait
-    - raw line
-    - just raw
 - url path support
 - refactor project structure