crawler

type Crawler struct {
	BaseURL                    string
	KeywordsToExclude          []string
	KeywordsToInclude          []string
	MaxNumberWorkers           int
	MaxNumberConnections       int
	Verbose                    bool
	FilePrefix                 string
	Remote, Username, Password string // Parameters for BoltDB remote connection
	UserAgent                  string
	TimeIntervalToPrintStats   int
	// contains filtered or unexported fields
}