From ecac18a0b7d314052db0be3d8c417911e8364f8f Mon Sep 17 00:00:00 2001 From: BroodjeAap Date: Sat, 4 Feb 2023 12:41:05 +0000 Subject: [PATCH] added broserless /function support --- scraping.go | 77 ++++++++++++++++++++++ static/edit.js | 134 +++++++++++++++++++++++--------------- static/edit.ts | 173 +++++++++++++++++++++++++++++++------------------ todo.md | 17 ----- 4 files changed, 271 insertions(+), 130 deletions(-) diff --git a/scraping.go b/scraping.go index 9a58d81..734b26b 100644 --- a/scraping.go +++ b/scraping.go @@ -244,6 +244,14 @@ func getFilterResult(filters []Filter, filter *Filter, watch *Watch, web *Web, d { getFilterResultBrowserlessURLs(filter, web.urlCache, debug) } + case "func": + { + getBrowserlessFunctionResult(filter) + } + case "funcs": + { + getBrowserlessFunctionResults(filter) + } } } case filter.Type == "echo": @@ -384,6 +392,7 @@ func getBrowserlessURLContent(filter *Filter, fetchURL string) (string, error) { filter.log("Could not marshal url:", err) return "", err } + browserlessURL = browserlessURL + "/content" resp, err := http.Post(browserlessURL, "application/json", bytes.NewBuffer(jsn)) if err != nil { log.Println("Could not get browserless response content:", err) @@ -399,6 +408,74 @@ func getBrowserlessURLContent(filter *Filter, fetchURL string) (string, error) { return string(body), nil } +func getBrowserlessFunctionResult(filter *Filter) { + result, err := getBrowserlessFunctionContent(filter, "") + if err != nil { + log.Println(err) + filter.log(err) + return + } + filter.Results = append(filter.Results, result) +} + +func getBrowserlessFunctionResults(filter *Filter) { + for _, parent := range filter.Parents { + for _, result := range parent.Results { + fetchURL := result + + str, err := getBrowserlessFunctionContent(filter, fetchURL) + if err != nil { + log.Println(err) + filter.log(err) + continue + } + filter.Results = append(filter.Results, str) + } + } +} + +type BrowserlessContext struct { + Result string `json:"result"` +} + +func getBrowserlessFunctionContent(filter *Filter, result string) (string, error) { + if !viper.IsSet("browserless.url") { + + return "", errors.New("browserless.url not set") + } + browserlessURL := viper.GetString("browserless.url") + if filter.Var2 == nil { + return "", errors.New("filter.Var2 == nil") + } + code := *filter.Var2 + data := struct { + Code string `json:"code"` + Context BrowserlessContext `json:"context"` + Detached bool `json:"detached"` + }{ + Code: code, + Context: BrowserlessContext{Result: result}, + Detached: false, + } + jsn, err := json.Marshal(data) + if err != nil { + return "", err + } + + browserlessURL = browserlessURL + "/function" + resp, err := http.Post(browserlessURL, "application/json", bytes.NewBuffer(jsn)) + if err != nil { + return "", err + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + + return string(body), nil +} + func getFilterResultXPath(filter *Filter) { selectType := "node" if filter.Var2 != nil { diff --git a/static/edit.js b/static/edit.js index a986519..3d8ba84 100644 --- a/static/edit.js +++ b/static/edit.js @@ -626,54 +626,14 @@ function onTypeChange(node) { gurlsOption.value = "gurls"; gurlsOption.innerHTML = "Get URLs"; browserlessSelect.appendChild(gurlsOption); - var startOption = document.createElement("option"); - startOption.value = "start"; - startOption.innerHTML = "Start Session"; - browserlessSelect.appendChild(startOption); - var endOption = document.createElement("option"); - endOption.value = "end"; - endOption.innerHTML = "End Session"; - browserlessSelect.appendChild(endOption); - var gotoOption = document.createElement("option"); - gotoOption.value = "go"; - gotoOption.innerHTML = "Goto URL"; - browserlessSelect.appendChild(gotoOption); - var cssOption = document.createElement("option"); - cssOption.value = "css"; - cssOption.innerHTML = "Click CSS Element"; - browserlessSelect.appendChild(cssOption); - var xpathOption = document.createElement("option"); - xpathOption.value = "xpath"; - xpathOption.innerHTML = "Click XPath Element"; - browserlessSelect.appendChild(xpathOption); - var authOption = document.createElement("option"); - authOption.value = "auth"; - authOption.innerHTML = "Authenticate"; - browserlessSelect.appendChild(authOption); - var selectOption = document.createElement("option"); - selectOption.value = "select"; - selectOption.innerHTML = "Set Dropdown"; - browserlessSelect.appendChild(selectOption); - var inputOption = document.createElement("option"); - inputOption.value = "input"; - inputOption.innerHTML = "Set Input"; - browserlessSelect.appendChild(inputOption); - var keyOption = document.createElement("option"); - keyOption.value = "key"; - keyOption.innerHTML = "Press Key"; - browserlessSelect.appendChild(keyOption); - var waitOption = document.createElement("option"); - waitOption.value = "wait"; - waitOption.innerHTML = "Wait"; - browserlessSelect.appendChild(waitOption); - var lineOption = document.createElement("option"); - lineOption.value = "line"; - lineOption.innerHTML = "Raw Line"; - browserlessSelect.appendChild(lineOption); - var rawOption = document.createElement("option"); - rawOption.value = "raw"; - rawOption.innerHTML = "Raw Function"; - browserlessSelect.appendChild(rawOption); + var funcOption = document.createElement("option"); + funcOption.value = "func"; + funcOption.innerHTML = "Function"; + browserlessSelect.appendChild(funcOption); + var funcsOption = document.createElement("option"); + funcsOption.value = "funcs"; + funcsOption.innerHTML = "Function on results"; + browserlessSelect.appendChild(funcsOption); var1Div.appendChild(browserlessSelect); if (var1Value == "") { browserlessSelect.value = "gurl"; @@ -754,7 +714,8 @@ function onTypeChange(node) { } finally { if (e_2) throw e_2.error; } } - var1Div.appendChild(snippetDiv); + var2Label.innerHTML = "Snippets"; + var2Div.appendChild(snippetDiv); var var2Input = document.createElement("input"); var2Input.name = "var2"; var2Input.id = "var2Input"; @@ -820,7 +781,6 @@ function onConditionChange(node) { var var1Value = ""; var var2Value = ""; var var3Value = ""; - console.log(node, var1Input.value); if (node != null) { // @ts-ignore var1Value = node.meta.var1; @@ -903,10 +863,10 @@ function onBrowserlessChange(node) { var var3Input = document.getElementById("var3Input"); var var3Label = document.getElementById("var3Label"); var var3Div = document.getElementById("var3Div"); + var3Div.innerHTML = ""; var var1Value = ""; var var2Value = ""; var var3Value = ""; - console.log(node, var1Input.value); if (node != null) { // @ts-ignore var1Value = node.meta.var1; @@ -932,6 +892,78 @@ function onBrowserlessChange(node) { break; } case "gurls": { + var var2Input_4 = document.createElement("input"); + var2Input_4.name = "var2"; + var2Input_4.id = "var2Input"; + var2Input_4.value = ""; + var2Input_4.disabled = true; + var2Input_4.classList.add("form-control"); + var2Label.innerHTML = "-"; + var2Div.appendChild(var2Input_4); + break; + } + case "func": { + var var2Input_5 = document.createElement("textarea"); + var2Input_5.name = "var2Input"; + var2Input_5.id = "var2Input"; + var2Input_5.value = "module.exports = async ({ page, context }) => {\n await page.goto(\"https://192.168.178.254:8000\");\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};"; + var2Input_5.classList.add("form-control"); + var2Input_5.rows = 15; + var2Label.innerHTML = "Code"; + var2Div.appendChild(var2Input_5); + if (var2Value != "") { + var2Input_5.value = var2Value; + } + var var3Input_1 = document.createElement("input"); + var3Input_1.type = "hidden"; + var3Input_1.id = "var3Input"; + var3Input_1.name = "var3Input"; + var3Div.appendChild(var3Input_1); + var3Label.innerHTML = "Help"; + var helpLink1 = document.createElement("a"); + helpLink1.href = "https://www.browserless.io/docs/function"; + helpLink1.innerHTML = "Browserless /Funcion"; + helpLink1.target = "_blank"; + helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink1); + var helpLink2 = document.createElement("a"); + helpLink2.href = "https://pptr.dev/api/puppeteer.page"; + helpLink2.innerHTML = "Puppeteer Page"; + helpLink2.target = "_blank"; + helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink2); + break; + } + case "funcs": { + var var2Input_6 = document.createElement("textarea"); + var2Input_6.name = "var2Input"; + var2Input_6.id = "var2Input"; + var2Input_6.value = "module.exports = async ({ page, context }) => {\n const { result } = context;\n await page.goto(result);\n\n const data = await page.content();\n\n return {\n data,\n type: 'text/plain', // 'application/html' 'application/json'\n };\n};"; + var2Input_6.classList.add("form-control"); + var2Input_6.rows = 15; + var2Label.innerHTML = "Code"; + var2Div.appendChild(var2Input_6); + if (var2Value != "") { + var2Input_6.value = var2Value; + } + var var3Input_2 = document.createElement("input"); + var3Input_2.type = "hidden"; + var3Input_2.id = "var3Input"; + var3Input_2.name = "var3Input"; + var3Div.appendChild(var3Input_2); + var3Label.innerHTML = "Help"; + var helpLink1 = document.createElement("a"); + helpLink1.href = "https://www.browserless.io/docs/function"; + helpLink1.innerHTML = "Browserless /Funcion"; + helpLink1.target = "_blank"; + helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink1); + var helpLink2 = document.createElement("a"); + helpLink2.href = "https://pptr.dev/api/puppeteer.page"; + helpLink2.innerHTML = "Puppeteer Page"; + helpLink2.target = "_blank"; + helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink2); break; } } diff --git a/static/edit.ts b/static/edit.ts index edaa53f..265c895 100644 --- a/static/edit.ts +++ b/static/edit.ts @@ -614,65 +614,15 @@ function onTypeChange(node: DiagramNode | null = null){ gurlsOption.innerHTML = "Get URLs"; browserlessSelect.appendChild(gurlsOption); - let startOption = document.createElement("option"); - startOption.value = "start" - startOption.innerHTML = "Start Session"; - browserlessSelect.appendChild(startOption); + let funcOption = document.createElement("option"); + funcOption.value = "func" + funcOption.innerHTML = "Function"; + browserlessSelect.appendChild(funcOption); - let endOption = document.createElement("option"); - endOption.value = "end" - endOption.innerHTML = "End Session"; - browserlessSelect.appendChild(endOption); - - let gotoOption = document.createElement("option"); - gotoOption.value = "go" - gotoOption.innerHTML = "Goto URL"; - browserlessSelect.appendChild(gotoOption); - - let cssOption = document.createElement("option"); - cssOption.value = "css" - cssOption.innerHTML = "Click CSS Element"; - browserlessSelect.appendChild(cssOption); - - let xpathOption = document.createElement("option"); - xpathOption.value = "xpath" - xpathOption.innerHTML = "Click XPath Element"; - browserlessSelect.appendChild(xpathOption); - - let authOption = document.createElement("option"); - authOption.value = "auth" - authOption.innerHTML = "Authenticate"; - browserlessSelect.appendChild(authOption); - - let selectOption = document.createElement("option"); - selectOption.value = "select" - selectOption.innerHTML = "Set Dropdown"; - browserlessSelect.appendChild(selectOption); - - let inputOption = document.createElement("option"); - inputOption.value = "input" - inputOption.innerHTML = "Set Input"; - browserlessSelect.appendChild(inputOption); - - let keyOption = document.createElement("option"); - keyOption.value = "key" - keyOption.innerHTML = "Press Key"; - browserlessSelect.appendChild(keyOption); - - let waitOption = document.createElement("option"); - waitOption.value = "wait" - waitOption.innerHTML = "Wait"; - browserlessSelect.appendChild(waitOption); - - let lineOption = document.createElement("option"); - lineOption.value = "line" - lineOption.innerHTML = "Raw Line"; - browserlessSelect.appendChild(lineOption); - - let rawOption = document.createElement("option"); - rawOption.value = "raw" - rawOption.innerHTML = "Raw Function"; - browserlessSelect.appendChild(rawOption); + let funcsOption = document.createElement("option"); + funcsOption.value = "funcs" + funcsOption.innerHTML = "Function on results"; + browserlessSelect.appendChild(funcsOption); var1Div.appendChild(browserlessSelect); @@ -885,7 +835,8 @@ end gap.classList.add("m-1", "xs"); snippetDiv.appendChild(gap); } - var1Div.appendChild(snippetDiv); + var2Label.innerHTML = "Snippets"; + var2Div.appendChild(snippetDiv); let var2Input = document.createElement("input"); var2Input.name = "var2"; @@ -954,7 +905,6 @@ function onConditionChange(node: DiagramNode | null = null){ let var1Value = ""; let var2Value = ""; let var3Value = ""; - console.log(node, var1Input.value) if (node != null){ // @ts-ignore var1Value = node.meta.var1; @@ -1029,11 +979,11 @@ function onBrowserlessChange(node: DiagramNode | null = null){ let var3Input = document.getElementById("var3Input") as HTMLInputElement; let var3Label = document.getElementById("var3Label") as HTMLLabelElement; let var3Div = document.getElementById("var3Div") as HTMLDivElement; + var3Div.innerHTML = ""; let var1Value = ""; let var2Value = ""; let var3Value = ""; - console.log(node, var1Input.value) if (node != null){ // @ts-ignore var1Value = node.meta.var1; @@ -1056,10 +1006,109 @@ function onBrowserlessChange(node: DiagramNode | null = null){ var2Input.value = var2Value; var2Input.classList.add("form-control") var2Label.innerHTML = "URL"; - var2Div.appendChild(var2Input) + var2Div.appendChild(var2Input); break; } case "gurls": { + let var2Input = document.createElement("input"); + var2Input.name = "var2"; + var2Input.id = "var2Input"; + var2Input.value = ""; + var2Input.disabled = true; + var2Input.classList.add("form-control") + var2Label.innerHTML = "-"; + var2Div.appendChild(var2Input); + break; + } + case "func": { + let var2Input = document.createElement("textarea"); + var2Input.name = "var2Input"; + var2Input.id = "var2Input"; + var2Input.value = `module.exports = async ({ page, context }) => { + await page.goto("https://192.168.178.254:8000"); + + const data = await page.content(); + + return { + data, + type: 'text/plain', // 'application/html' 'application/json' + }; +};`; + var2Input.classList.add("form-control"); + var2Input.rows = 15; + var2Label.innerHTML = "Code"; + var2Div.appendChild(var2Input); + + if (var2Value != ""){ + var2Input.value = var2Value; + } + + let var3Input = document.createElement("input"); + var3Input.type = "hidden"; + var3Input.id = "var3Input"; + var3Input.name = "var3Input"; + var3Div.appendChild(var3Input); + + var3Label.innerHTML = "Help"; + let helpLink1 = document.createElement("a"); + helpLink1.href = "https://www.browserless.io/docs/function"; + helpLink1.innerHTML = "Browserless /Funcion"; + helpLink1.target = "_blank"; + helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink1); + + let helpLink2 = document.createElement("a"); + helpLink2.href = "https://pptr.dev/api/puppeteer.page"; + helpLink2.innerHTML = "Puppeteer Page"; + helpLink2.target = "_blank"; + helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink2); + break; + } + case "funcs":{ + let var2Input = document.createElement("textarea"); + var2Input.name = "var2Input"; + var2Input.id = "var2Input"; + var2Input.value = `module.exports = async ({ page, context }) => { + const { result } = context; + await page.goto(result); + + const data = await page.content(); + + return { + data, + type: 'text/plain', // 'application/html' 'application/json' + }; +};`; + var2Input.classList.add("form-control"); + var2Input.rows = 15; + var2Label.innerHTML = "Code"; + var2Div.appendChild(var2Input); + + if (var2Value != ""){ + var2Input.value = var2Value; + } + + let var3Input = document.createElement("input"); + var3Input.type = "hidden"; + var3Input.id = "var3Input"; + var3Input.name = "var3Input"; + var3Div.appendChild(var3Input); + + var3Label.innerHTML = "Help"; + let helpLink1 = document.createElement("a"); + helpLink1.href = "https://www.browserless.io/docs/function"; + helpLink1.innerHTML = "Browserless /Funcion"; + helpLink1.target = "_blank"; + helpLink1.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink1); + + let helpLink2 = document.createElement("a"); + helpLink2.href = "https://pptr.dev/api/puppeteer.page"; + helpLink2.innerHTML = "Puppeteer Page"; + helpLink2.target = "_blank"; + helpLink2.classList.add("btn", "btn-outline-secondary", "btn-sm", "xs"); + var3Div.appendChild(helpLink2); break; } } diff --git a/todo.md b/todo.md index bb93240..3930e27 100644 --- a/todo.md +++ b/todo.md @@ -5,22 +5,5 @@ - util.go - edit.ts - diagram.ts -- browserless function filters - - ~~get url~~ - - ~~get urls~~ - - session - - start - - end - - goto url - - click elem - - xpath - - css - - basic auth - - set select - - set text input - - press key - - wait - - raw line - - just raw - url path support - refactor project structure \ No newline at end of file