A Step by Step Guide to Build a Trend Finder Tool with Python: Web Scraping, NLP (Sentiment Analysis & Topic Modeling), and Word Cloud Visualization

import requests
from bs4 import BeautifulSoup


# List of URLs to scrape
urls = ["https://en.wikipedia.org/wiki/Natural_language_processing",
        "https://en.wikipedia.org/wiki/Machine_learning"]  


collected_texts = []  # to store text from each page


for url in urls:
    response = requests.get(url, headers={"User-Agent": "Mozilla/5.0"})
    if response.status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        # Extract all paragraph text
        paragraphs = [p.get_text() for p in soup.find_all('p')]
        page_text = " ".join(paragraphs)
        collected_texts.append(page_text.strip())
    else:
        print(f"Failed to retrieve {url}")

import re
import nltk
nltk.download('stopwords')
from nltk.corpus import stopwords


stop_words = set(stopwords.words('english'))


cleaned_texts = []
for text in collected_texts:
    # Remove non-alphabetical characters and lower the text
    text = re.sub(r'[^A-Za-z\s]', ' ', text).lower()
    # Remove stopwords
    words = [w for w in text.split() if w not in stop_words]
    cleaned_texts.append(" ".join(words))

from collections import Counter


# Combine all texts into one if analyzing overall trends:
all_text = " ".join(cleaned_texts)
word_counts = Counter(all_text.split())
common_words = word_counts.most_common(10)  # top 10 frequent words
print("Top 10 keywords:", common_words)

!pip install textblob
from textblob import TextBlob


for i, text in enumerate(cleaned_texts, 1):
    polarity = TextBlob(text).sentiment.polarity
    if polarity > 0.1:
        sentiment = "Positive 😀"
    elif polarity < -0.1:
        sentiment = "Negative 🙁"
    else:
        sentiment = "Neutral 😐"
    print(f"Document {i} Sentiment: {sentiment} (polarity={polarity:.2f})")

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.decomposition import LatentDirichletAllocation


# Adjust these parameters
vectorizer = CountVectorizer(max_df=1.0, min_df=1, stop_words="english")
doc_term_matrix = vectorizer.fit_transform(cleaned_texts)


# Fit LDA to find topics (for instance, 3 topics)
lda = LatentDirichletAllocation(n_components=3, random_state=42)
lda.fit(doc_term_matrix)


feature_names = vectorizer.get_feature_names_out()


for idx, topic in enumerate(lda.components_):
    print(f"Topic {idx + 1}: ", [vectorizer.get_feature_names_out()[i] for i in topic.argsort()[:-11:-1]])

# Assuming you have your text data stored in combined_text
from wordcloud import WordCloud
import matplotlib.pyplot as plt
import nltk
from nltk.corpus import stopwords
import re


nltk.download('stopwords')
stop_words = set(stopwords.words('english'))


# Preprocess and clean the text:
cleaned_texts = []
for text in collected_texts:
    text = re.sub(r'[^A-Za-z\s]', ' ', text).lower()
    words = [w for w in text.split() if w not in stop_words]
    cleaned_texts.append(" ".join(words))


# Generate combined text
combined_text = " ".join(cleaned_texts)


# Generate the word cloud
wordcloud = WordCloud(width=800, height=400, background_color="white", colormap='viridis').generate(combined_text)


# Display the word cloud
plt.figure(figsize=(10, 6))  # <-- corrected numeric dimensions
plt.imshow(wordcloud, interpolation='bilinear')
plt.axis('off')
plt.title("Word Cloud of Scraped Text", fontsize=16)
plt.show()