爬虫开发

.env.example

@@ -61,4 +61,19 @@ AWS_USE_PATH_STYLE_ENDPOINT=false
 
 VITE_APP_NAME="${APP_NAME}"
 
-
+CRAWLER_USER_AGENT="AIWebCrawler/1.0 (+https://dev.aiweb.com)"
+CRAWLER_REQUEST_TIMEOUT=20
+CRAWLER_AI_TIMEOUT=60
+CRAWLER_VERIFY_SSL=true
+CRAWLER_DNS_SERVERS=
+CRAWLER_FORCE_IPV4=false
+CRAWLER_AI_WIRE_API=chat_completions
+CRAWLER_AI_BASE_URL=
+CRAWLER_BROWSERLESS_ENDPOINT=
+CRAWLER_BROWSERLESS_TOKEN=
+CRAWLER_AI_ENDPOINT=
+CRAWLER_AI_KEY=
+CRAWLER_AI_MODEL=gpt-4o-mini
+CRAWLER_AI_REASONING_EFFORT=
+CRAWLER_AI_DISABLE_RESPONSE_STORAGE=false
+CRAWLER_ALERT_EMAIL=