sidski/helseveileder_webscraper
Archived
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

🎨 add config file

Browse source 
Co-authored-by: haraldnilsen <harald_998@hotmail.com>
Signed-off-by: Sindre Kjelsrud <kjelsrudsindre@gmail.com>
This commit is contained in:
Sindre Kjelsrud 2024-01-08 15:14:05 +01:00
parent 709937da9d
commit 86a4b18a59
Signed by untrusted user who does not match committer: sidski
GPG key ID: D2BBDF3EDE6BA9A6
2 changed files with 6 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
config.py Normal file
View file

@ -0,0 +1,3 @@
headers = {
"User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"
}

10
main.py
View file

@ -2,13 +2,11 @@ import httpx
from selectolax.parser import HTMLParser from selectolax.parser import HTMLParser
import re import re
import csv import csv
import config
def fetch_question_url(url): def fetch_question_url(url):
# Define headers # Define headers
headers = { headers = config.headers
"User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"
}
# Define the parameters for the request # Define the parameters for the request
params = { params = {
"action": "get-faqs", "action": "get-faqs",
@ -40,9 +38,7 @@ def fetch_question_url(url):
def fetch_all_info(url): def fetch_all_info(url):
# Define headers # Define headers
headers = { headers = config.headers
"User-Agent":"Mozilla/5.0 (X11; Linux x86_64; rv:121.0) Gecko/20100101 Firefox/121.0"
}
response = httpx.get(url, headers=headers) response = httpx.get(url, headers=headers)
html = HTMLParser(response.text) html = HTMLParser(response.text)