156 lines
4.1 KiB
Go
156 lines
4.1 KiB
Go
package main
|
|
|
|
import (
|
|
"fmt"
|
|
"regexp"
|
|
"time"
|
|
"github.com/go-rod/rod"
|
|
"github.com/schollz/progressbar/v3"
|
|
)
|
|
|
|
var (
|
|
visitedURLs = make(map[string]bool)
|
|
allLinks = make(map[string]bool)
|
|
maxDepth = 3 // Hier können Sie die maximale Tiefe festlegen
|
|
|
|
// Neue Variablen für initiale URLs und Regex-Pattern
|
|
initialURLs = []string{"https://pad.stratum0.org/p/dc"}
|
|
linkRegexPattern = `https://pad\.stratum0\.org/p/dc[^\s"']+`
|
|
)
|
|
|
|
func main() {
|
|
fmt.Println("Starte das Programm...")
|
|
startTime := time.Now()
|
|
|
|
browser := rod.New().MustConnect()
|
|
defer browser.MustClose()
|
|
|
|
fmt.Printf("Initiale URLs: %v\n", initialURLs)
|
|
fmt.Printf("Link Regex Pattern: %s\n", linkRegexPattern)
|
|
fmt.Printf("Maximale Tiefe: %d\n", maxDepth)
|
|
|
|
toVisit := make([]struct {
|
|
url string
|
|
depth int
|
|
}, len(initialURLs))
|
|
|
|
for i, url := range initialURLs {
|
|
toVisit[i] = struct {
|
|
url string
|
|
depth int
|
|
}{url, 0}
|
|
}
|
|
|
|
totalURLs := len(toVisit)
|
|
bar := progressbar.Default(int64(totalURLs))
|
|
|
|
for len(toVisit) > 0 {
|
|
current := toVisit[0]
|
|
toVisit = toVisit[1:]
|
|
|
|
newLinks := extractLinksFromPage(browser, current.url, current.depth)
|
|
|
|
for _, link := range newLinks {
|
|
if !visitedURLs[link] && current.depth < maxDepth {
|
|
toVisit = append(toVisit, struct {
|
|
url string
|
|
depth int
|
|
}{link, current.depth + 1})
|
|
totalURLs++
|
|
bar.ChangeMax(totalURLs)
|
|
}
|
|
}
|
|
|
|
bar.Add(1)
|
|
}
|
|
|
|
fmt.Println("\nAlle gefundenen Links:")
|
|
for link := range allLinks {
|
|
fmt.Println(link)
|
|
}
|
|
|
|
fmt.Printf("\nStatistik:\n")
|
|
fmt.Printf("Gesamtanzahl der gefundenen Links: %d\n", len(allLinks))
|
|
fmt.Printf("Anzahl der besuchten URLs: %d\n", len(visitedURLs))
|
|
fmt.Printf("Gesamtzeit: %v\n", time.Since(startTime))
|
|
}
|
|
|
|
func extractLinksFromPage(browser *rod.Browser, url string, depth int) []string {
|
|
fmt.Printf("\nVerarbeite URL: %s (Tiefe: %d)\n", url, depth)
|
|
|
|
if depth > maxDepth {
|
|
fmt.Printf("Maximale Tiefe erreicht für URL: %s\n", url)
|
|
return nil
|
|
}
|
|
|
|
if visitedURLs[url] {
|
|
fmt.Printf("URL bereits besucht: %s\n", url)
|
|
return nil
|
|
}
|
|
|
|
visitedURLs[url] = true
|
|
|
|
page := browser.MustPage(url)
|
|
defer page.MustClose()
|
|
page.MustWaitLoad()
|
|
|
|
fmt.Printf("Seite geladen: %s\n", url)
|
|
|
|
bar := progressbar.Default(100)
|
|
|
|
var newLinks []string
|
|
|
|
mainLinks := extractLinks(page, url)
|
|
newLinks = append(newLinks, mainLinks...)
|
|
bar.Add(50)
|
|
|
|
iframeLinks := processNestedIframes(page, url)
|
|
newLinks = append(newLinks, iframeLinks...)
|
|
bar.Add(50)
|
|
|
|
return newLinks
|
|
}
|
|
|
|
func processNestedIframes(page *rod.Page, sourceURL string) []string {
|
|
fmt.Printf("Suche nach äußerem iFrame auf %s\n", sourceURL)
|
|
|
|
outerIframeElement := page.MustElement("#editorcontainer > iframe:nth-child(1)")
|
|
outerFrame := outerIframeElement.MustFrame()
|
|
outerFrame.MustWaitLoad()
|
|
|
|
fmt.Printf("Äußeres iFrame geladen auf %s\n", sourceURL)
|
|
|
|
outerLinks := extractLinks(outerFrame, sourceURL+" (äußeres iFrame)")
|
|
|
|
fmt.Printf("Suche nach innerem iFrame auf %s\n", sourceURL)
|
|
|
|
innerIframeElement := outerFrame.MustElement("#outerdocbody > iframe:nth-child(1)")
|
|
innerFrame := innerIframeElement.MustFrame()
|
|
innerFrame.MustWaitLoad()
|
|
|
|
fmt.Printf("Inneres iFrame geladen auf %s\n", sourceURL)
|
|
|
|
innerLinks := extractLinks(innerFrame, sourceURL+" (inneres iFrame)")
|
|
|
|
return append(outerLinks, innerLinks...)
|
|
}
|
|
|
|
func extractLinks(page *rod.Page, sourceURL string) []string {
|
|
text := page.MustElement("body").MustText()
|
|
|
|
re := regexp.MustCompile(linkRegexPattern)
|
|
links := re.FindAllString(text, -1)
|
|
|
|
fmt.Printf("Gefundene Links auf %s: %d\n", sourceURL, len(links))
|
|
|
|
var newLinks []string
|
|
for _, link := range links {
|
|
if !allLinks[link] {
|
|
allLinks[link] = true
|
|
fmt.Printf("Neuer Link gefunden: %s\n", link)
|
|
newLinks = append(newLinks, link)
|
|
}
|
|
}
|
|
|
|
return newLinks
|
|
}
|