Dfyjkt

I need to create the dataset on news. I need to extract all the news of given news website which have ever been posted on that website. I have write this code

import requests 
from bs4 import BeautifulSoup 
import pandas 
import csv 
from datetime import datetime

records=

def cnbc(base_url):

 r = requests.get(base_url) 
 c = r.content 
 soup = BeautifulSoup(c,"html.parser") 
 Title=soup.find("h1","class":"title"}).text.replace("r","").replace("n","")
 content=' '
 for content_tag in soup.find_all("p"):
 content = content+content_tag.text.replace("r","").replace("n","")
 content= content[18:-458]
 Country ='United States'
 website='https://www.cnbc.com/' 
 comments='' 
 genre='Political'
 date= soup.find("time","class":"datestamp").text[35:-2].replace("r","").replace("n","")
 d = datetime.strptime(date, "%d %b %Y")
 date = d.strftime("%d-%m-%Y")
 records.append((Title,content,Country,website,comments,genre,date))

cnbc("https://www.cnbc.com/2018/11/02/here-are-the-three-things-pulling-down-the-stock-market-again.html")

but this is only allowing me to extract one news.

Can anyone tell me that how can I extract all the news url from the root directory of website.

This is python3 script and it is not flawless but I hope it can serve as starting point so you can achieve what you are trying to. I am not sure does this site from which you are trying to scrape data allows such operation so I will not place their web address for constants WEB_SITE_BASE_URL and WEB_SITE_REGION_URL. It is your choice what you are going to put there.

import requests 
from bs4 import BeautifulSoup 
from datetime import datetime

# https://www.xxxx.com"
WEB_SITE_BASE_URL= ""
# https://www.xxxx.com/?region=us
WEB_SITE_REGION_URL = ""

def get_categories(web_site_base_url):
 r = requests.get(web_site_base_url) 
 c = r.content 
 soup = BeautifulSoup(c,"html.parser") 
 spans = soup.find_all(attrs="nav-menu-buttonText")
 categories = [category.text for category in spans]
 return categories

def get_links(category_url):
 r = requests.get(category_url)
 c = r.content 
 soup = BeautifulSoup(c,"html.parser")
 links = [a.get('href') for a in soup.find_all('a', href=True)]
 filtered_links = list(set([k for k in links if '/2018/11/' in k]))
 return filtered_links

def news(link):
 r = requests.get(link) 
 c = r.content 
 soup = BeautifulSoup(c,"html.parser") 
 Title=soup.find("h1","class":"title").text.replace("r","").replace("n","")
 content=' '
 for content_tag in soup.find_all("p"):
 content = content+content_tag.text.replace("r","").replace("n","")
 content= content[18:-458]
 Country ='United States'
 website='WEB_SITE_BASE_URL' 
 comments=''
 date= soup.find("time","class":"datestamp").text[35:-2].replace("r","").replace("n","")
 d = datetime.strptime(date, "%d %b %Y")
 date = d.strftime("%d-%m-%Y") 
 spans = soup.find_all(attrs="header_title last breadcrumb")
 categories = [category.text for category in spans]
 genre = categories
 return(Title,content,Country,website,comments,genre,date)

categories = get_categories(WEB_SITE_REGION_URL)
list_of_link_lists = 
for category in categories:
 list_of_link_lists.append(get_links(WEB_SITE_BASE_URL + "/" + category.replace(" ", "20%")))
flat_link_list = list(set([item for sublist in list_of_link_lists for item in sublist]))
articles_list = 
for link in flat_link_list:
 try:
 articles_list.append(news(WEB_SITE_BASE_URL + link))
 except:
 print("Something was wrong")
 continue

print(articles_list)

There is a rough method to extract part of all news, the method showed as my code. First, extract all the news which div class is headline news_headline = soup.find_all("div",class_="headline"). Then check if element is what we want.

new = 
for div in news_headline:
 each = ()
 if div.a:
 each[0] = url + div.a.get("href")
 if div.a.text:
 # use split to remove t n blankspace
 each[1] = " ".join(div.a.text.split())
 else:
 each[1] = " ".join(div.a.get("title").split())
 new.append(each)
 else:
 continue

It is the full code but i wrote this as short as i can.

import requests 
from bs4 import BeautifulSoup 

def index(url="https://www.cnbc.com/world/"):
 with requests.Session() as se:
 se.encoding = "UTF-8"
 res = se.get(url)
 text = res.text
 soup = BeautifulSoup(text,"lxml")
 news_headline = soup.find_all("div",class_="headline")
 news_ = [(url + div.a.get("href"), " ".join(div.a.text.split()) if div.a.text else "".join(div.a.get("title").split()) ) for div in news_headline if div.a]
 print(news_)

index()